@powersync/service-module-mongodb 0.0.0-dev-20241111122558 → 0.1.0

package/src/replication/ChangeStream.ts

@@ -11,6 +11,8 @@ import {
   mongoLsnToTimestamp
 } from './MongoRelation.js';
 import { escapeRegExp } from '../utils.js';
+import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
+import { PostImagesOption } from '../types/types.js';
 
 export const ZERO_LSN = '0000000000000000';
 
@@ -24,7 +26,15 @@ interface InitResult {
   needsInitialSync: boolean;
 }
 
-export class MissingReplicationSlotError extends Error {
+/**
+ * Thrown when the change stream is not valid anymore, and replication
+ * must be restarted.
+ *
+ * Possible reasons:
+ *  * Some change stream documents do not have postImages.
+ *  * startAfter/resumeToken is not valid anymore.
+ */
+export class ChangeStreamInvalidatedError extends Error {
   constructor(message: string) {
     super(message);
   }
@@ -70,7 +80,21 @@ export class ChangeStream {
     return this.abort_signal.aborted;
   }
 
-  async getQualifiedTableNames(
+  private get usePostImages() {
+    return this.connections.options.postImages != PostImagesOption.OFF;
+  }
+
+  private get configurePostImages() {
+    return this.connections.options.postImages == PostImagesOption.AUTO_CONFIGURE;
+  }
+
+  /**
+   * This resolves a pattern, persists the related metadata, and returns
+   * the resulting SourceTables.
+   *
+   * This implicitly checks the collection postImage configuration.
+   */
+  async resolveQualifiedTableNames(
     batch: storage.BucketStorageBatch,
     tablePattern: TablePattern
   ): Promise<storage.SourceTable[]> {
@@ -94,10 +118,14 @@ export class ChangeStream {
         {
           name: nameFilter
         },
-        { nameOnly: true }
+        { nameOnly: false }
       )
       .toArray();
 
+    if (!tablePattern.isWildcard && collections.length == 0) {
+      logger.warn(`Collection ${schema}.${tablePattern.name} not found`);
+    }
+
     for (let collection of collections) {
       const table = await this.handleRelation(
         batch,
@@ -108,7 +136,7 @@ export class ChangeStream {
           replicationColumns: [{ name: '_id' }]
         } as SourceEntityDescriptor,
         // This is done as part of the initial setup - snapshot is handled elsewhere
-        { snapshot: false }
+        { snapshot: false, collectionInfo: collection }
       );
 
       result.push(table);
@@ -148,44 +176,45 @@ export class ChangeStream {
     const sourceTables = this.sync_rules.getSourceTables();
     await this.client.connect();
 
+    // We need to get the snapshot time before taking the initial snapshot.
     const hello = await this.defaultDb.command({ hello: 1 });
-    const startTime = hello.lastWrite?.majorityOpTime?.ts as mongo.Timestamp;
+    const snapshotTime = hello.lastWrite?.majorityOpTime?.ts as mongo.Timestamp;
     if (hello.msg == 'isdbgrid') {
       throw new Error('Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
     } else if (hello.setName == null) {
       throw new Error('Standalone MongoDB instances are not supported - use a replicaset.');
-    } else if (startTime == null) {
+    } else if (snapshotTime == null) {
       // Not known where this would happen apart from the above cases
       throw new Error('MongoDB lastWrite timestamp not found.');
     }
-    const session = await this.client.startSession({
-      snapshot: true
-    });
+    // We previously used {snapshot: true} for the snapshot session.
+    // While it gives nice consistency guarantees, it fails when the
+    // snapshot takes longer than 5 minutes, due to minSnapshotHistoryWindowInSeconds
+    // expiring the snapshot.
+    const session = await this.client.startSession();
     try {
       await this.storage.startBatch(
-        { zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName },
+        { zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
         async (batch) => {
+          // Start by resolving all tables.
+          // This checks postImage configuration, and that should fail as
+          // earlier as possible.
+          let allSourceTables: SourceTable[] = [];
           for (let tablePattern of sourceTables) {
-            const tables = await this.getQualifiedTableNames(batch, tablePattern);
-            for (let table of tables) {
-              await this.snapshotTable(batch, table, session);
-              await batch.markSnapshotDone([table], ZERO_LSN);
-
-              await touch();
-            }
+            const tables = await this.resolveQualifiedTableNames(batch, tablePattern);
+            allSourceTables.push(...tables);
           }
 
-          const snapshotTime = session.clusterTime?.clusterTime ?? startTime;
+          for (let table of allSourceTables) {
+            await this.snapshotTable(batch, table, session);
+            await batch.markSnapshotDone([table], ZERO_LSN);
 
-          if (snapshotTime != null) {
-            const lsn = getMongoLsn(snapshotTime);
-            logger.info(`Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`);
-            // keepalive() does an auto-commit if there is data
-            await batch.flush();
-            await batch.keepalive(lsn);
-          } else {
-            throw new Error(`No snapshot clusterTime available.`);
+            await touch();
           }
+
+          const lsn = getMongoLsn(snapshotTime);
+          logger.info(`Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`);
+          await batch.commit(lsn);
         }
       );
     } finally {
@@ -193,10 +222,26 @@ export class ChangeStream {
     }
   }
 
+  private async setupCheckpointsCollection() {
+    const collection = await this.getCollectionInfo(this.defaultDb.databaseName, CHECKPOINTS_COLLECTION);
+    if (collection == null) {
+      await this.defaultDb.createCollection(CHECKPOINTS_COLLECTION, {
+        changeStreamPreAndPostImages: { enabled: true }
+      });
+    } else if (this.usePostImages && collection.options?.changeStreamPreAndPostImages?.enabled != true) {
+      // Drop + create requires less permissions than collMod,
+      // and we don't care about the data in this collection.
+      await this.defaultDb.dropCollection(CHECKPOINTS_COLLECTION);
+      await this.defaultDb.createCollection(CHECKPOINTS_COLLECTION, {
+        changeStreamPreAndPostImages: { enabled: true }
+      });
+    }
+  }
+
   private getSourceNamespaceFilters(): { $match: any; multipleDatabases: boolean } {
     const sourceTables = this.sync_rules.getSourceTables();
 
-    let $inFilters: any[] = [{ db: this.defaultDb.databaseName, coll: '_powersync_checkpoints' }];
+    let $inFilters: any[] = [{ db: this.defaultDb.databaseName, coll: CHECKPOINTS_COLLECTION }];
     let $refilters: any[] = [];
     let multipleDatabases = false;
     for (let tablePattern of sourceTables) {
@@ -209,7 +254,10 @@ export class ChangeStream {
       }
 
       if (tablePattern.isWildcard) {
-        $refilters.push({ db: tablePattern.schema, coll: new RegExp('^' + escapeRegExp(tablePattern.tablePrefix)) });
+        $refilters.push({
+          'ns.db': tablePattern.schema,
+          'ns.coll': new RegExp('^' + escapeRegExp(tablePattern.tablePrefix))
+        });
       } else {
         $inFilters.push({
           db: tablePattern.schema,
@@ -240,7 +288,7 @@ export class ChangeStream {
 
     const db = this.client.db(table.schema);
     const collection = db.collection(table.table);
-    const query = collection.find({}, { session });
+    const query = collection.find({}, { session, readConcern: { level: 'majority' } });
 
     const cursor = query.stream();
 
@@ -249,6 +297,8 @@ export class ChangeStream {
         throw new Error(`Aborted initial replication`);
       }
 
+      at += 1;
+
       const record = constructAfterRecord(document);
 
       // This auto-flushes when the batch reaches its size limit
@@ -268,6 +318,7 @@ export class ChangeStream {
     }
 
     await batch.flush();
+    logger.info(`Replicated ${at} documents for ${table.qualifiedName}`);
   }
 
   private async getRelation(
@@ -278,14 +329,60 @@ export class ChangeStream {
     if (existing != null) {
       return existing;
     }
-    return this.handleRelation(batch, descriptor, { snapshot: false });
+
+    // Note: collection may have been dropped at this point, so we handle
+    // missing values.
+    const collection = await this.getCollectionInfo(descriptor.schema, descriptor.name);
+
+    return this.handleRelation(batch, descriptor, { snapshot: false, collectionInfo: collection });
+  }
+
+  private async getCollectionInfo(db: string, name: string): Promise<mongo.CollectionInfo | undefined> {
+    const collection = (
+      await this.client
+        .db(db)
+        .listCollections(
+          {
+            name: name
+          },
+          { nameOnly: false }
+        )
+        .toArray()
+    )[0];
+    return collection;
+  }
+
+  private async checkPostImages(db: string, collectionInfo: mongo.CollectionInfo) {
+    if (!this.usePostImages) {
+      // Nothing to check
+      return;
+    }
+
+    const enabled = collectionInfo.options?.changeStreamPreAndPostImages?.enabled == true;
+
+    if (!enabled && this.configurePostImages) {
+      await this.client.db(db).command({
+        collMod: collectionInfo.name,
+        changeStreamPreAndPostImages: { enabled: true }
+      });
+      logger.info(`Enabled postImages on ${db}.${collectionInfo.name}`);
+    } else if (!enabled) {
+      throw new Error(`postImages not enabled on ${db}.${collectionInfo.name}`);
+    }
   }
 
   async handleRelation(
     batch: storage.BucketStorageBatch,
     descriptor: SourceEntityDescriptor,
-    options: { snapshot: boolean }
+    options: { snapshot: boolean; collectionInfo: mongo.CollectionInfo | undefined }
   ) {
+    if (options.collectionInfo != null) {
+      await this.checkPostImages(descriptor.schema, options.collectionInfo);
+    } else {
+      // If collectionInfo is null, the collection may have been dropped.
+      // Ignore the postImages check in this case.
+    }
+
     const snapshot = options.snapshot;
     if (!descriptor.objectId && typeof descriptor.objectId != 'string') {
       throw new Error('objectId expected');
@@ -388,121 +485,180 @@ export class ChangeStream {
 
   async initReplication() {
     const result = await this.initSlot();
+    await this.setupCheckpointsCollection();
     if (result.needsInitialSync) {
       await this.startInitialReplication();
     }
   }
 
   async streamChanges() {
+    try {
+      await this.streamChangesInternal();
+    } catch (e) {
+      if (
+        e instanceof mongo.MongoServerError &&
+        e.codeName == 'NoMatchingDocument' &&
+        e.errmsg?.includes('post-image was not found')
+      ) {
+        throw new ChangeStreamInvalidatedError(e.errmsg);
+      }
+      throw e;
+    }
+  }
+
+  async streamChangesInternal() {
     // Auto-activate as soon as initial replication is done
     await this.storage.autoActivate();
 
-    await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName }, async (batch) => {
-      const lastLsn = batch.lastCheckpointLsn;
-      const startAfter = mongoLsnToTimestamp(lastLsn) ?? undefined;
-      logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`);
-
-      // TODO: Use changeStreamSplitLargeEvent
-
-      const filters = this.getSourceNamespaceFilters();
-
-      const pipeline: mongo.Document[] = [
-        {
-          $match: filters.$match
+    await this.storage.startBatch(
+      { zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
+      async (batch) => {
+        const lastLsn = batch.lastCheckpointLsn;
+        const startAfter = mongoLsnToTimestamp(lastLsn) ?? undefined;
+        logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`);
+
+        const filters = this.getSourceNamespaceFilters();
+
+        const pipeline: mongo.Document[] = [
+          {
+            $match: filters.$match
+          },
+          { $changeStreamSplitLargeEvent: {} }
+        ];
+
+        let fullDocument: 'required' | 'updateLookup';
+
+        if (this.usePostImages) {
+          // 'read_only' or 'auto_configure'
+          // Configuration happens during snapshot, or when we see new
+          // collections.
+          fullDocument = 'required';
+        } else {
+          fullDocument = 'updateLookup';
         }
-      ];
-
-      const streamOptions: mongo.ChangeStreamOptions = {
-        startAtOperationTime: startAfter,
-        showExpandedEvents: true,
-        useBigInt64: true,
-        maxAwaitTimeMS: 200,
-        fullDocument: 'updateLookup'
-      };
-      let stream: mongo.ChangeStream<mongo.Document>;
-      if (filters.multipleDatabases) {
-        // Requires readAnyDatabase@admin on Atlas
-        stream = this.client.watch(pipeline, streamOptions);
-      } else {
-        // Same general result, but requires less permissions than the above
-        stream = this.defaultDb.watch(pipeline, streamOptions);
-      }
 
-      if (this.abort_signal.aborted) {
-        stream.close();
-        return;
-      }
-
-      this.abort_signal.addEventListener('abort', () => {
-        stream.close();
-      });
-
-      let waitForCheckpointLsn: string | null = null;
+        const streamOptions: mongo.ChangeStreamOptions = {
+          startAtOperationTime: startAfter,
+          showExpandedEvents: true,
+          useBigInt64: true,
+          maxAwaitTimeMS: 200,
+          fullDocument: fullDocument
+        };
+        let stream: mongo.ChangeStream<mongo.Document>;
+        if (filters.multipleDatabases) {
+          // Requires readAnyDatabase@admin on Atlas
+          stream = this.client.watch(pipeline, streamOptions);
+        } else {
+          // Same general result, but requires less permissions than the above
+          stream = this.defaultDb.watch(pipeline, streamOptions);
+        }
 
-      while (true) {
         if (this.abort_signal.aborted) {
-          break;
+          stream.close();
+          return;
         }
 
-        const changeDocument = await stream.tryNext();
-
-        if (changeDocument == null || this.abort_signal.aborted) {
-          continue;
-        }
-        await touch();
+        this.abort_signal.addEventListener('abort', () => {
+          stream.close();
+        });
 
-        if (startAfter != null && changeDocument.clusterTime?.lte(startAfter)) {
-          continue;
-        }
+        // Always start with a checkpoint.
+        // This helps us to clear erorrs when restarting, even if there is
+        // no data to replicate.
+        let waitForCheckpointLsn: string | null = await createCheckpoint(this.client, this.defaultDb);
 
-        // console.log('event', changeDocument);
+        let splitDocument: mongo.ChangeStreamDocument | null = null;
 
-        if (
-          (changeDocument.operationType == 'insert' ||
-            changeDocument.operationType == 'update' ||
-            changeDocument.operationType == 'replace') &&
-          changeDocument.ns.coll == '_powersync_checkpoints'
-        ) {
-          const lsn = getMongoLsn(changeDocument.clusterTime!);
-          if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
-            waitForCheckpointLsn = null;
+        while (true) {
+          if (this.abort_signal.aborted) {
+            break;
           }
-          await batch.flush();
-          await batch.keepalive(lsn);
-        } else if (
-          changeDocument.operationType == 'insert' ||
-          changeDocument.operationType == 'update' ||
-          changeDocument.operationType == 'replace' ||
-          changeDocument.operationType == 'delete'
-        ) {
-          if (waitForCheckpointLsn == null) {
-            waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb);
+
+          const originalChangeDocument = await stream.tryNext();
+
+          if (originalChangeDocument == null || this.abort_signal.aborted) {
+            continue;
           }
-          const rel = getMongoRelation(changeDocument.ns);
-          const table = await this.getRelation(batch, rel);
-          if (table.syncAny) {
-            await this.writeChange(batch, table, changeDocument);
+          await touch();
+
+          if (startAfter != null && originalChangeDocument.clusterTime?.lte(startAfter)) {
+            continue;
           }
-        } else if (changeDocument.operationType == 'drop') {
-          const rel = getMongoRelation(changeDocument.ns);
-          const table = await this.getRelation(batch, rel);
-          if (table.syncAny) {
-            await batch.drop([table]);
-            this.relation_cache.delete(table.objectId);
+
+          let changeDocument = originalChangeDocument;
+          if (originalChangeDocument?.splitEvent != null) {
+            // Handle split events from $changeStreamSplitLargeEvent.
+            // This is only relevant for very large update operations.
+            const splitEvent = originalChangeDocument?.splitEvent;
+
+            if (splitDocument == null) {
+              splitDocument = originalChangeDocument;
+            } else {
+              splitDocument = Object.assign(splitDocument, originalChangeDocument);
+            }
+
+            if (splitEvent.fragment == splitEvent.of) {
+              // Got all fragments
+              changeDocument = splitDocument;
+              splitDocument = null;
+            } else {
+              // Wait for more fragments
+              continue;
+            }
+          } else if (splitDocument != null) {
+            // We were waiting for fragments, but got a different event
+            throw new Error(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
           }
-        } else if (changeDocument.operationType == 'rename') {
-          const relFrom = getMongoRelation(changeDocument.ns);
-          const relTo = getMongoRelation(changeDocument.to);
-          const tableFrom = await this.getRelation(batch, relFrom);
-          if (tableFrom.syncAny) {
-            await batch.drop([tableFrom]);
-            this.relation_cache.delete(tableFrom.objectId);
+
+          // console.log('event', changeDocument);
+
+          if (
+            (changeDocument.operationType == 'insert' ||
+              changeDocument.operationType == 'update' ||
+              changeDocument.operationType == 'replace') &&
+            changeDocument.ns.coll == CHECKPOINTS_COLLECTION
+          ) {
+            const lsn = getMongoLsn(changeDocument.clusterTime!);
+            if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
+              waitForCheckpointLsn = null;
+            }
+            await batch.commit(lsn);
+          } else if (
+            changeDocument.operationType == 'insert' ||
+            changeDocument.operationType == 'update' ||
+            changeDocument.operationType == 'replace' ||
+            changeDocument.operationType == 'delete'
+          ) {
+            if (waitForCheckpointLsn == null) {
+              waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb);
+            }
+            const rel = getMongoRelation(changeDocument.ns);
+            const table = await this.getRelation(batch, rel);
+            if (table.syncAny) {
+              await this.writeChange(batch, table, changeDocument);
+            }
+          } else if (changeDocument.operationType == 'drop') {
+            const rel = getMongoRelation(changeDocument.ns);
+            const table = await this.getRelation(batch, rel);
+            if (table.syncAny) {
+              await batch.drop([table]);
+              this.relation_cache.delete(table.objectId);
+            }
+          } else if (changeDocument.operationType == 'rename') {
+            const relFrom = getMongoRelation(changeDocument.ns);
+            const relTo = getMongoRelation(changeDocument.to);
+            const tableFrom = await this.getRelation(batch, relFrom);
+            if (tableFrom.syncAny) {
+              await batch.drop([tableFrom]);
+              this.relation_cache.delete(tableFrom.objectId);
+            }
+            // Here we do need to snapshot the new table
+            const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
+            await this.handleRelation(batch, relTo, { snapshot: true, collectionInfo: collection });
           }
-          // Here we do need to snapshot the new table
-          await this.handleRelation(batch, relTo, { snapshot: true });
         }
       }
-    });
+    );
   }
 }
 

package/src/replication/ChangeStreamReplicationJob.ts

@@ -1,5 +1,5 @@
 import { container } from '@powersync/lib-services-framework';
-import { MissingReplicationSlotError, ChangeStream } from './ChangeStream.js';
+import { ChangeStreamInvalidatedError, ChangeStream } from './ChangeStream.js';
 
 import { replication } from '@powersync/service-core';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
@@ -40,7 +40,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
       });
       this.logger.error(`Replication failed`, e);
 
-      if (e instanceof MissingReplicationSlotError) {
+      if (e instanceof ChangeStreamInvalidatedError) {
         // This stops replication on this slot, and creates a new slot
         await this.options.storage.factory.slotRemoved(this.slotName);
       }
@@ -84,8 +84,10 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
         // Without this additional log, the cause may not be visible in the logs.
         this.logger.error(`cause`, e.cause);
       }
-      if (e instanceof mongo.MongoError && e.hasErrorLabel('NonResumableChangeStreamError')) {
-        throw new MissingReplicationSlotError(e.message);
+      if (e instanceof ChangeStreamInvalidatedError) {
+        throw e;
+      } else if (e instanceof mongo.MongoError && e.hasErrorLabel('NonResumableChangeStreamError')) {
+        throw new ChangeStreamInvalidatedError(e.message);
       } else {
         // Report the error if relevant, before retrying
         container.reporter.captureException(e, {

package/src/replication/MongoErrorRateLimiter.ts

@@ -1,5 +1,5 @@
-import { setTimeout } from 'timers/promises';
 import { ErrorRateLimiter } from '@powersync/service-core';
+import { setTimeout } from 'timers/promises';
 
 export class MongoErrorRateLimiter implements ErrorRateLimiter {
   nextAllowed: number = Date.now();
@@ -27,13 +27,6 @@ export class MongoErrorRateLimiter implements ErrorRateLimiter {
     } else if (message.includes('ECONNREFUSED')) {
       // Could be fail2ban or similar
       this.setDelay(120_000);
-    } else if (
-      message.includes('Unable to do postgres query on ended pool') ||
-      message.includes('Postgres unexpectedly closed connection')
-    ) {
-      // Connection timed out - ignore / immediately retry
-      // We don't explicitly set the delay to 0, since there could have been another error that
-      // we need to respect.
     } else {
       this.setDelay(30_000);
     }

package/src/replication/MongoRelation.ts

@@ -2,6 +2,7 @@ import { storage } from '@powersync/service-core';
 import { SqliteRow, SqliteValue, toSyncRulesRow } from '@powersync/service-sync-rules';
 import * as mongo from 'mongodb';
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
+import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
 
 export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor {
   return {
@@ -145,7 +146,10 @@ function filterJsonData(data: any, depth = 0): any {
 export async function createCheckpoint(client: mongo.MongoClient, db: mongo.Db): Promise<string> {
   const session = client.startSession();
   try {
-    const result = await db.collection('_powersync_checkpoints').findOneAndUpdate(
+    // Note: If multiple PowerSync instances are replicating the same source database,
+    // they'll modify the same checkpoint document. This is fine - it could create
+    // more replication load than required, but won't break anything.
+    await db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
       {
         _id: 'checkpoint' as any
       },
@@ -159,7 +163,6 @@ export async function createCheckpoint(client: mongo.MongoClient, db: mongo.Db):
       }
     );
     const time = session.operationTime!;
-    // console.log('marked checkpoint at', time, getMongoLsn(time));
     // TODO: Use the above when we support custom write checkpoints
     return getMongoLsn(time);
   } finally {

package/src/replication/replication-utils.ts

@@ -1,6 +1,7 @@
-import * as mongo from 'mongodb';
 import { MongoManager } from './MongoManager.js';
 
+export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
+
 export async function checkSourceConfiguration(connectionManager: MongoManager): Promise<void> {
   const db = connectionManager.db;
   const hello = await db.command({ hello: 1 });