@powersync/service-module-mongodb 0.0.0-dev-20250117095455 → 0.0.0-dev-20250214100224

package/src/module/MongoModule.ts

@@ -1,5 +1,12 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { api, ConfigurationFileSyncRulesProvider, replication, system, TearDownOptions } from '@powersync/service-core';
+import {
+  api,
+  ConfigurationFileSyncRulesProvider,
+  ConnectionTestResult,
+  replication,
+  system,
+  TearDownOptions
+} from '@powersync/service-core';
 import { MongoRouteAPIAdapter } from '../api/MongoRouteAPIAdapter.js';
 import { ChangeStreamReplicator } from '../replication/ChangeStreamReplicator.js';
 import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactory.js';
@@ -46,22 +53,31 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
   }
 
   async teardown(options: TearDownOptions): Promise<void> {
-    // TODO: Implement?
+    // No-op
   }
 
-  async testConnection(config: types.MongoConnectionConfig): Promise<void> {
+  async testConnection(config: types.MongoConnectionConfig) {
     this.decodeConfig(config);
-    const normalisedConfig = this.resolveConfig(this.decodedConfig!);
-    const connectionManager = new MongoManager(normalisedConfig, {
+    const normalizedConfig = this.resolveConfig(this.decodedConfig!);
+    return await MongoModule.testConnection(normalizedConfig);
+  }
+
+  static async testConnection(normalizedConfig: types.NormalizedMongoConnectionConfig): Promise<ConnectionTestResult> {
+    const connectionManager = new MongoManager(normalizedConfig, {
       // Use short timeouts for testing connections.
       // Must be < 30s, to ensure we get a proper timeout error.
-      socketTimeoutMS: 5_000,
-      serverSelectionTimeoutMS: 5_000
+      socketTimeoutMS: 1_000,
+      serverSelectionTimeoutMS: 1_000
     });
     try {
-      return await checkSourceConfiguration(connectionManager);
+      await checkSourceConfiguration(connectionManager);
+    } catch (e) {
+      throw lib_mongo.mapConnectionError(e);
     } finally {
       await connectionManager.end();
     }
+    return {
+      connectionDescription: normalizedConfig.uri
+    };
   }
 }

package/src/replication/ChangeStream.ts

@@ -1,21 +1,22 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { container, logger } from '@powersync/lib-services-framework';
+import {
+  container,
+  DatabaseConnectionError,
+  ErrorCode,
+  logger,
+  ReplicationAbortedError,
+  ReplicationAssertionError,
+  ServiceError
+} from '@powersync/lib-services-framework';
 import { Metrics, SaveOperationTag, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core';
 import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import { MongoLSN } from '../common/MongoLSN.js';
 import { PostImagesOption } from '../types/types.js';
 import { escapeRegExp } from '../utils.js';
 import { MongoManager } from './MongoManager.js';
-import {
-  constructAfterRecord,
-  createCheckpoint,
-  getMongoLsn,
-  getMongoRelation,
-  mongoLsnToTimestamp
-} from './MongoRelation.js';
+import { constructAfterRecord, createCheckpoint, getMongoRelation } from './MongoRelation.js';
 import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
 
-export const ZERO_LSN = '0000000000000000';
-
 export interface ChangeStreamOptions {
   connections: MongoManager;
   storage: storage.SyncRulesBucketStorage;
@@ -34,9 +35,9 @@ interface InitResult {
  *  * Some change stream documents do not have postImages.
  *  * startAfter/resumeToken is not valid anymore.
  */
-export class ChangeStreamInvalidatedError extends Error {
-  constructor(message: string) {
-    super(message);
+export class ChangeStreamInvalidatedError extends DatabaseConnectionError {
+  constructor(message: string, cause: any) {
+    super(ErrorCode.PSYNC_S1344, message, cause);
   }
 }
 
@@ -157,7 +158,7 @@ export class ChangeStream {
 
   async estimatedCount(table: storage.SourceTable): Promise<string> {
     const db = this.client.db(table.schema);
-    const count = db.collection(table.table).estimatedDocumentCount();
+    const count = await db.collection(table.table).estimatedDocumentCount();
     return `~${count}`;
   }
 
@@ -180,12 +181,18 @@ export class ChangeStream {
     const hello = await this.defaultDb.command({ hello: 1 });
     const snapshotTime = hello.lastWrite?.majorityOpTime?.ts as mongo.Timestamp;
     if (hello.msg == 'isdbgrid') {
-      throw new Error('Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1341,
+        'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
+      );
     } else if (hello.setName == null) {
-      throw new Error('Standalone MongoDB instances are not supported - use a replicaset.');
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1342,
+        'Standalone MongoDB instances are not supported - use a replicaset.'
+      );
     } else if (snapshotTime == null) {
       // Not known where this would happen apart from the above cases
-      throw new Error('MongoDB lastWrite timestamp not found.');
+      throw new ReplicationAssertionError('MongoDB lastWrite timestamp not found.');
     }
     // We previously used {snapshot: true} for the snapshot session.
     // While it gives nice consistency guarantees, it fails when the
@@ -194,7 +201,7 @@ export class ChangeStream {
     const session = await this.client.startSession();
     try {
       await this.storage.startBatch(
-        { zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
+        { zeroLSN: MongoLSN.ZERO.comparable, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
         async (batch) => {
           // Start by resolving all tables.
           // This checks postImage configuration, and that should fail as
@@ -207,12 +214,12 @@ export class ChangeStream {
 
           for (let table of allSourceTables) {
             await this.snapshotTable(batch, table, session);
-            await batch.markSnapshotDone([table], ZERO_LSN);
+            await batch.markSnapshotDone([table], MongoLSN.ZERO.comparable);
 
             await touch();
           }
 
-          const lsn = getMongoLsn(snapshotTime);
+          const { comparable: lsn } = new MongoLSN({ timestamp: snapshotTime });
           logger.info(`Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`);
           await batch.commit(lsn);
         }
@@ -285,6 +292,7 @@ export class ChangeStream {
     logger.info(`Replicating ${table.qualifiedName}`);
     const estimatedCount = await this.estimatedCount(table);
     let at = 0;
+    let lastLogIndex = 0;
 
     const db = this.client.db(table.schema);
     const collection = db.collection(table.table);
@@ -294,11 +302,9 @@ export class ChangeStream {
 
     for await (let document of cursor) {
       if (this.abort_signal.aborted) {
-        throw new Error(`Aborted initial replication`);
+        throw new ReplicationAbortedError(`Aborted initial replication`);
       }
 
-      at += 1;
-
       const record = constructAfterRecord(document);
 
       // This auto-flushes when the batch reaches its size limit
@@ -312,6 +318,10 @@ export class ChangeStream {
       });
 
       at += 1;
+      if (at - lastLogIndex >= 5000) {
+        logger.info(`[${this.group_id}] Replicating ${table.qualifiedName} ${at}/${estimatedCount}`);
+        lastLogIndex = at;
+      }
       Metrics.getInstance().rows_replicated_total.add(1);
 
       await touch();
@@ -367,7 +377,7 @@ export class ChangeStream {
       });
       logger.info(`Enabled postImages on ${db}.${collectionInfo.name}`);
     } else if (!enabled) {
-      throw new Error(`postImages not enabled on ${db}.${collectionInfo.name}`);
+      throw new ServiceError(ErrorCode.PSYNC_S1343, `postImages not enabled on ${db}.${collectionInfo.name}`);
     }
   }
 
@@ -385,7 +395,7 @@ export class ChangeStream {
 
     const snapshot = options.snapshot;
     if (!descriptor.objectId && typeof descriptor.objectId != 'string') {
-      throw new Error('objectId expected');
+      throw new ReplicationAssertionError('MongoDB replication - objectId expected');
     }
     const result = await this.storage.resolveTable({
       group_id: this.group_id,
@@ -466,7 +476,7 @@ export class ChangeStream {
         beforeReplicaId: change.documentKey._id
       });
     } else {
-      throw new Error(`Unsupported operation: ${change.operationType}`);
+      throw new ReplicationAssertionError(`Unsupported operation: ${change.operationType}`);
     }
   }
 
@@ -500,7 +510,7 @@ export class ChangeStream {
         e.codeName == 'NoMatchingDocument' &&
         e.errmsg?.includes('post-image was not found')
       ) {
-        throw new ChangeStreamInvalidatedError(e.errmsg);
+        throw new ChangeStreamInvalidatedError(e.errmsg, e);
       }
       throw e;
     }
@@ -511,10 +521,13 @@ export class ChangeStream {
     await this.storage.autoActivate();
 
     await this.storage.startBatch(
-      { zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
+      { zeroLSN: MongoLSN.ZERO.comparable, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
       async (batch) => {
-        const lastLsn = batch.lastCheckpointLsn;
-        const startAfter = mongoLsnToTimestamp(lastLsn) ?? undefined;
+        const { lastCheckpointLsn } = batch;
+        const lastLsn = lastCheckpointLsn ? MongoLSN.fromSerialized(lastCheckpointLsn) : null;
+        const startAfter = lastLsn?.timestamp;
+        const resumeAfter = lastLsn?.resumeToken;
+
         logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`);
 
         const filters = this.getSourceNamespaceFilters();
@@ -538,12 +551,21 @@ export class ChangeStream {
         }
 
         const streamOptions: mongo.ChangeStreamOptions = {
-          startAtOperationTime: startAfter,
           showExpandedEvents: true,
           useBigInt64: true,
           maxAwaitTimeMS: 200,
           fullDocument: fullDocument
         };
+
+        /**
+         * Only one of these options can be supplied at a time.
+         */
+        if (resumeAfter) {
+          streamOptions.resumeAfter = resumeAfter;
+        } else {
+          streamOptions.startAtOperationTime = startAfter;
+        }
+
         let stream: mongo.ChangeStream<mongo.Document>;
         if (filters.multipleDatabases) {
           // Requires readAnyDatabase@admin on Atlas
@@ -563,7 +585,7 @@ export class ChangeStream {
         });
 
         // Always start with a checkpoint.
-        // This helps us to clear erorrs when restarting, even if there is
+        // This helps us to clear errors when restarting, even if there is
         // no data to replicate.
         let waitForCheckpointLsn: string | null = await createCheckpoint(this.client, this.defaultDb);
 
@@ -576,6 +598,11 @@ export class ChangeStream {
 
           const originalChangeDocument = await stream.tryNext();
 
+          // The stream was closed, we will only ever receive `null` from it
+          if (!originalChangeDocument && stream.closed) {
+            break;
+          }
+
           if (originalChangeDocument == null || this.abort_signal.aborted) {
             continue;
           }
@@ -607,18 +634,41 @@ export class ChangeStream {
             }
           } else if (splitDocument != null) {
             // We were waiting for fragments, but got a different event
-            throw new Error(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+            throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
           }
 
-          // console.log('event', changeDocument);
-
           if (
             (changeDocument.operationType == 'insert' ||
               changeDocument.operationType == 'update' ||
-              changeDocument.operationType == 'replace') &&
+              changeDocument.operationType == 'replace' ||
+              changeDocument.operationType == 'drop') &&
             changeDocument.ns.coll == CHECKPOINTS_COLLECTION
           ) {
-            const lsn = getMongoLsn(changeDocument.clusterTime!);
+            /**
+             * Dropping the database does not provide an `invalidate` event.
+             * We typically would receive `drop` events for the collection which we
+             * would process below.
+             *
+             * However we don't commit the LSN after collections are dropped.
+             * The prevents the `startAfter` or `resumeToken` from advancing past the drop events.
+             * The stream also closes after the drop events.
+             * This causes an infinite loop of processing the collection drop events.
+             *
+             * This check here invalidates the change stream if our `_checkpoints` collection
+             * is dropped. This allows for detecting when the DB is dropped.
+             */
+            if (changeDocument.operationType == 'drop') {
+              throw new ChangeStreamInvalidatedError(
+                'Internal collections have been dropped',
+                new Error('_checkpoints collection was dropped')
+              );
+            }
+
+            const { comparable: lsn } = new MongoLSN({
+              timestamp: changeDocument.clusterTime!,
+              resume_token: changeDocument._id
+            });
+
             if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
               waitForCheckpointLsn = null;
             }

package/src/replication/ChangeStreamReplicationJob.ts

@@ -1,4 +1,4 @@
-import { mongo } from '@powersync/lib-service-mongodb';
+import { isMongoServerError } from '@powersync/lib-service-mongodb';
 import { container } from '@powersync/lib-services-framework';
 import { replication } from '@powersync/service-core';
 
@@ -78,15 +78,15 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
       if (this.abortController.signal.aborted) {
         return;
       }
-      this.logger.error(`Replication error`, e);
+      this.logger.error(`${this.slotName} Replication error`, e);
       if (e.cause != null) {
         // Without this additional log, the cause may not be visible in the logs.
         this.logger.error(`cause`, e.cause);
       }
       if (e instanceof ChangeStreamInvalidatedError) {
         throw e;
-      } else if (e instanceof mongo.MongoError && e.hasErrorLabel('NonResumableChangeStreamError')) {
-        throw new ChangeStreamInvalidatedError(e.message);
+      } else if (isMongoServerError(e) && e.hasErrorLabel('NonResumableChangeStreamError')) {
+        throw new ChangeStreamInvalidatedError(e.message, e);
       } else {
         // Report the error if relevant, before retrying
         container.reporter.captureException(e, {

package/src/replication/ChangeStreamReplicator.ts

@@ -2,6 +2,7 @@ import { storage, replication } from '@powersync/service-core';
 import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
 import { MongoErrorRateLimiter } from './MongoErrorRateLimiter.js';
+import { MongoModule } from '../module/MongoModule.js';
 
 export interface ChangeStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
   connectionFactory: ConnectionManagerFactory;
@@ -33,4 +34,8 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator<Chang
     await super.stop();
     await this.connectionFactory.shutdown();
   }
+
+  async testConnection() {
+    return await MongoModule.testConnection(this.connectionFactory.dbConnectionConfig);
+  }
 }

package/src/replication/ConnectionManagerFactory.ts

@@ -4,7 +4,7 @@ import { MongoManager } from './MongoManager.js';
 
 export class ConnectionManagerFactory {
   private readonly connectionManagers: MongoManager[];
-  private readonly dbConnectionConfig: NormalizedMongoConnectionConfig;
+  public readonly dbConnectionConfig: NormalizedMongoConnectionConfig;
 
   constructor(dbConnectionConfig: NormalizedMongoConnectionConfig) {
     this.dbConnectionConfig = dbConnectionConfig;

package/src/replication/MongoRelation.ts

@@ -3,6 +3,8 @@ import { storage } from '@powersync/service-core';
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
 import { SqliteRow, SqliteValue } from '@powersync/service-sync-rules';
 
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { MongoLSN } from '../common/MongoLSN.js';
 import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
 
 export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor {
@@ -14,21 +16,6 @@ export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.S
   } satisfies storage.SourceEntityDescriptor;
 }
 
-export function getMongoLsn(timestamp: mongo.Timestamp) {
-  const a = timestamp.high.toString(16).padStart(8, '0');
-  const b = timestamp.low.toString(16).padStart(8, '0');
-  return a + b;
-}
-
-export function mongoLsnToTimestamp(lsn: string | null) {
-  if (lsn == null) {
-    return null;
-  }
-  const a = parseInt(lsn.substring(0, 8), 16);
-  const b = parseInt(lsn.substring(8, 16), 16);
-  return mongo.Timestamp.fromBits(b, a);
-}
-
 export function constructAfterRecord(document: mongo.Document): SqliteRow {
   let record: SqliteRow = {};
   for (let key of Object.keys(document)) {
@@ -97,7 +84,7 @@ function filterJsonData(data: any, depth = 0): any {
   const autoBigNum = true;
   if (depth > DEPTH_LIMIT) {
     // This is primarily to prevent infinite recursion
-    throw new Error(`json nested object depth exceeds the limit of ${DEPTH_LIMIT}`);
+    throw new ServiceError(ErrorCode.PSYNC_S1004, `json nested object depth exceeds the limit of ${DEPTH_LIMIT}`);
   }
   if (data === null) {
     return data;
@@ -173,7 +160,7 @@ export async function createCheckpoint(client: mongo.MongoClient, db: mongo.Db):
     );
     const time = session.operationTime!;
     // TODO: Use the above when we support custom write checkpoints
-    return getMongoLsn(time);
+    return new MongoLSN({ timestamp: time }).comparable;
   } finally {
     await session.endSession();
   }

package/src/replication/replication-utils.ts

@@ -1,13 +1,88 @@
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { MongoManager } from './MongoManager.js';
+import { PostImagesOption } from '../types/types.js';
 
 export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
 
+const REQUIRED_CHECKPOINT_PERMISSIONS = ['find', 'insert', 'update', 'remove', 'changeStream', 'createCollection'];
+
 export async function checkSourceConfiguration(connectionManager: MongoManager): Promise<void> {
   const db = connectionManager.db;
+
   const hello = await db.command({ hello: 1 });
   if (hello.msg == 'isdbgrid') {
-    throw new Error('Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1341,
+      'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
+    );
   } else if (hello.setName == null) {
-    throw new Error('Standalone MongoDB instances are not supported - use a replicaset.');
+    throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
+  }
+
+  // https://www.mongodb.com/docs/manual/reference/command/connectionStatus/
+  const connectionStatus = await db.command({ connectionStatus: 1, showPrivileges: true });
+  const priviledges = connectionStatus.authInfo?.authenticatedUserPrivileges as {
+    resource: { db: string; collection: string };
+    actions: string[];
+  }[];
+  let checkpointsActions = new Set<string>();
+  let anyCollectionActions = new Set<string>();
+  if (priviledges?.length > 0) {
+    const onDefaultDb = priviledges.filter((p) => p.resource.db == db.databaseName || p.resource.db == '');
+    const onCheckpoints = onDefaultDb.filter(
+      (p) => p.resource.collection == CHECKPOINTS_COLLECTION || p.resource?.collection == ''
+    );
+
+    for (let p of onCheckpoints) {
+      for (let a of p.actions) {
+        checkpointsActions.add(a);
+      }
+    }
+    for (let p of onDefaultDb) {
+      for (let a of p.actions) {
+        anyCollectionActions.add(a);
+      }
+    }
+
+    const missingCheckpointActions = REQUIRED_CHECKPOINT_PERMISSIONS.filter(
+      (action) => !checkpointsActions.has(action)
+    );
+    if (missingCheckpointActions.length > 0) {
+      const fullName = `${db.databaseName}.${CHECKPOINTS_COLLECTION}`;
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1307,
+        `MongoDB user does not have the required ${missingCheckpointActions.map((a) => `"${a}"`).join(', ')} priviledge(s) on "${fullName}".`
+      );
+    }
+
+    if (connectionManager.options.postImages == PostImagesOption.AUTO_CONFIGURE) {
+      // This checks that we have collMod on _any_ collection in the db.
+      // This is not a complete check, but does give a basic sanity-check for testing the connection.
+      if (!anyCollectionActions.has('collMod')) {
+        throw new ServiceError(
+          ErrorCode.PSYNC_S1307,
+          `MongoDB user does not have the required "collMod" priviledge on "${db.databaseName}", required for "post_images: auto_configure".`
+        );
+      }
+    }
+    if (!anyCollectionActions.has('listCollections')) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1307,
+        `MongoDB user does not have the required "listCollections" priviledge on "${db.databaseName}".`
+      );
+    }
+  } else {
+    // Assume auth is disabled.
+    // On Atlas, at least one role/priviledge is required for each user, which will trigger the above.
+
+    // We do still do a basic check that we can list the collection (it may not actually exist yet).
+    await db
+      .listCollections(
+        {
+          name: CHECKPOINTS_COLLECTION
+        },
+        { nameOnly: false }
+      )
+      .toArray();
   }
 }

package/test/src/change_stream_utils.ts

@@ -58,7 +58,7 @@ export class ChangeStreamTestContext {
   }
 
   async updateSyncRules(content: string) {
-    const syncRules = await this.factory.updateSyncRules({ content: content });
+    const syncRules = await this.factory.updateSyncRules({ content: content, validate: true });
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
@@ -85,7 +85,7 @@ export class ChangeStreamTestContext {
   }
 
   startStreaming() {
-    this.streamPromise = this.walStream.streamChanges();
+    return (this.streamPromise = this.walStream.streamChanges());
   }
 
   async getCheckpoint(options?: { timeout?: number }) {