@powersync/service-module-mongodb-storage 0.9.4 → 0.10.0

package/src/storage/implementation/MongoBucketBatch.ts

@@ -7,11 +7,19 @@ import {
   container,
   ErrorCode,
   errors,
-  logger,
+  Logger,
+  logger as defaultLogger,
   ReplicationAssertionError,
   ServiceError
 } from '@powersync/lib-services-framework';
-import { deserializeBson, InternalOpId, SaveOperationTag, storage, utils } from '@powersync/service-core';
+import {
+  BucketStorageMarkRecordUnavailable,
+  deserializeBson,
+  InternalOpId,
+  SaveOperationTag,
+  storage,
+  utils
+} from '@powersync/service-core';
 import * as timers from 'node:timers/promises';
 import { PowerSyncMongo } from './db.js';
 import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
@@ -46,12 +54,18 @@ export interface MongoBucketBatchOptions {
    * Set to true for initial replication.
    */
   skipExistingRows: boolean;
+
+  markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
+
+  logger?: Logger;
 }
 
 export class MongoBucketBatch
   extends BaseObserver<storage.BucketBatchStorageListener>
   implements storage.BucketStorageBatch
 {
+  private logger: Logger;
+
   private readonly client: mongo.MongoClient;
   public readonly db: PowerSyncMongo;
   public readonly session: mongo.ClientSession;
@@ -65,6 +79,7 @@ export class MongoBucketBatch
 
   private batch: OperationBatch | null = null;
   private write_checkpoint_batch: storage.CustomWriteCheckpointOptions[] = [];
+  private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
 
   /**
    * Last LSN received associated with a checkpoint.
@@ -86,6 +101,7 @@ export class MongoBucketBatch
 
   constructor(options: MongoBucketBatchOptions) {
     super();
+    this.logger = options.logger ?? defaultLogger;
     this.client = options.db.client;
     this.db = options.db;
     this.group_id = options.groupId;
@@ -96,6 +112,7 @@ export class MongoBucketBatch
     this.sync_rules = options.syncRules;
     this.storeCurrentData = options.storeCurrentData;
     this.skipExistingRows = options.skipExistingRows;
+    this.markRecordUnavailable = options.markRecordUnavailable;
     this.batch = new OperationBatch();
 
     this.persisted_op = options.keepaliveOp ?? null;
@@ -112,32 +129,34 @@ export class MongoBucketBatch
     return this.last_checkpoint_lsn;
   }
 
-  async flush(): Promise<storage.FlushedResult | null> {
+  async flush(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null> {
     let result: storage.FlushedResult | null = null;
     // One flush may be split over multiple transactions.
     // Each flushInner() is one transaction.
-    while (this.batch != null) {
-      let r = await this.flushInner();
+    while (this.batch != null || this.write_checkpoint_batch.length > 0) {
+      let r = await this.flushInner(options);
       if (r) {
         result = r;
       }
     }
-    await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch);
-    this.write_checkpoint_batch = [];
     return result;
   }
 
-  private async flushInner(): Promise<storage.FlushedResult | null> {
+  private async flushInner(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null> {
     const batch = this.batch;
-    if (batch == null) {
-      return null;
-    }
-
     let last_op: InternalOpId | null = null;
     let resumeBatch: OperationBatch | null = null;
 
-    await this.withReplicationTransaction(`Flushing ${batch.length} ops`, async (session, opSeq) => {
-      resumeBatch = await this.replicateBatch(session, batch, opSeq);
+    await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
+      if (batch != null) {
+        resumeBatch = await this.replicateBatch(session, batch, opSeq, options);
+      }
+
+      if (this.write_checkpoint_batch.length > 0) {
+        this.logger.info(`Writing ${this.write_checkpoint_batch.length} custom write checkpoints`);
+        await batchCreateCustomWriteCheckpoints(this.db, session, this.write_checkpoint_batch, opSeq.next());
+        this.write_checkpoint_batch = [];
+      }
 
       last_op = opSeq.last();
     });
@@ -157,7 +176,8 @@ export class MongoBucketBatch
   private async replicateBatch(
     session: mongo.ClientSession,
     batch: OperationBatch,
-    op_seq: MongoIdSequence
+    op_seq: MongoIdSequence,
+    options?: storage.BucketBatchCommitOptions
   ): Promise<OperationBatch | null> {
     let sizes: Map<string, number> | undefined = undefined;
     if (this.storeCurrentData && !this.skipExistingRows) {
@@ -231,7 +251,9 @@ export class MongoBucketBatch
         current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc);
       }
 
-      let persistedBatch: PersistedBatch | null = new PersistedBatch(this.group_id, transactionSize);
+      let persistedBatch: PersistedBatch | null = new PersistedBatch(this.group_id, transactionSize, {
+        logger: this.logger
+      });
 
       for (let op of b) {
         if (resumeBatch) {
@@ -253,7 +275,7 @@ export class MongoBucketBatch
         if (persistedBatch!.shouldFlushTransaction()) {
           // Transaction is getting big.
           // Flush, and resume in a new transaction.
-          await persistedBatch!.flush(this.db, this.session);
+          await persistedBatch!.flush(this.db, this.session, options);
           persistedBatch = null;
           // Computing our current progress is a little tricky here, since
           // we're stopping in the middle of a batch.
@@ -264,7 +286,7 @@ export class MongoBucketBatch
 
       if (persistedBatch) {
         transactionSize = persistedBatch.currentSize;
-        await persistedBatch.flush(this.db, this.session);
+        await persistedBatch.flush(this.db, this.session, options);
       }
     }
 
@@ -310,11 +332,18 @@ export class MongoBucketBatch
         // Not an error if we re-apply a transaction
         existing_buckets = [];
         existing_lookups = [];
-        // Log to help with debugging if there was a consistency issue
         if (this.storeCurrentData) {
-          logger.warn(
-            `Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
-          );
+          if (this.markRecordUnavailable != null) {
+            // This will trigger a "resnapshot" of the record.
+            // This is not relevant if storeCurrentData is false, since we'll get the full row
+            // directly in the replication stream.
+            this.markRecordUnavailable(record);
+          } else {
+            // Log to help with debugging if there was a consistency issue
+            this.logger.warn(
+              `Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
+            );
+          }
         }
       } else {
         existing_buckets = result.buckets;
@@ -331,8 +360,8 @@ export class MongoBucketBatch
         existing_buckets = [];
         existing_lookups = [];
         // Log to help with debugging if there was a consistency issue
-        if (this.storeCurrentData) {
-          logger.warn(
+        if (this.storeCurrentData && this.markRecordUnavailable == null) {
+          this.logger.warn(
             `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
           );
         }
@@ -429,7 +458,7 @@ export class MongoBucketBatch
               }
             }
           );
-          logger.error(
+          this.logger.error(
             `Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`
           );
         }
@@ -469,7 +498,7 @@ export class MongoBucketBatch
               }
             }
           );
-          logger.error(
+          this.logger.error(
             `Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${after.id}: ${error.error}`
           );
         }
@@ -523,7 +552,7 @@ export class MongoBucketBatch
             if (e instanceof mongo.MongoError && e.hasErrorLabel('TransientTransactionError')) {
               // Likely write conflict caused by concurrent write stream replicating
             } else {
-              logger.warn('Transaction error', e as Error);
+              this.logger.warn('Transaction error', e as Error);
             }
             await timers.setTimeout(Math.random() * 50);
             throw e;
@@ -548,7 +577,7 @@ export class MongoBucketBatch
     await this.withTransaction(async () => {
       flushTry += 1;
       if (flushTry % 10 == 0) {
-        logger.info(`${this.slot_name} ${description} - try ${flushTry}`);
+        this.logger.info(`${description} - try ${flushTry}`);
       }
       if (flushTry > 20 && Date.now() > lastTry) {
         throw new ServiceError(ErrorCode.PSYNC_S1402, 'Max transaction tries exceeded');
@@ -600,6 +629,7 @@ export class MongoBucketBatch
         },
         { session }
       );
+      // We don't notify checkpoint here - we don't make any checkpoint updates directly
     });
   }
 
@@ -613,17 +643,18 @@ export class MongoBucketBatch
   async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<boolean> {
     const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
 
-    await this.flush();
+    await this.flush(options);
 
     if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
       // When re-applying transactions, don't create a new checkpoint until
       // we are past the last transaction.
-      logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`);
+      this.logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`);
+      // Cannot create a checkpoint yet - return false
       return false;
     }
     if (lsn < this.no_checkpoint_before_lsn) {
       if (Date.now() - this.lastWaitingLogThottled > 5_000) {
-        logger.info(
+        this.logger.info(
           `Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}`
         );
         this.lastWaitingLogThottled = Date.now();
@@ -646,12 +677,15 @@ export class MongoBucketBatch
         },
         { session: this.session }
       );
+      await this.db.notifyCheckpoint();
 
+      // Cannot create a checkpoint yet - return false
       return false;
     }
 
     if (!createEmptyCheckpoints && this.persisted_op == null) {
-      return false;
+      // Nothing to commit - also return true
+      return true;
     }
 
     const now = new Date();
@@ -668,15 +702,34 @@ export class MongoBucketBatch
       update.last_checkpoint = this.persisted_op;
     }
 
+    // Mark relevant write checkpoints as "processed".
+    // This makes it easier to identify write checkpoints that are "valid" in order.
+    await this.db.write_checkpoints.updateMany(
+      {
+        processed_at_lsn: null,
+        'lsns.1': { $lte: lsn }
+      },
+      {
+        $set: {
+          processed_at_lsn: lsn
+        }
+      },
+      {
+        session: this.session
+      }
+    );
+
     await this.db.sync_rules.updateOne(
       {
         _id: this.group_id
       },
       {
-        $set: update
+        $set: update,
+        $unset: { snapshot_lsn: 1 }
       },
       { session: this.session }
     );
+    await this.db.notifyCheckpoint();
     this.persisted_op = null;
     this.last_checkpoint_lsn = lsn;
     return true;
@@ -695,10 +748,25 @@ export class MongoBucketBatch
     if (this.persisted_op != null) {
       // The commit may have been skipped due to "no_checkpoint_before_lsn".
       // Apply it now if relevant
-      logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`);
+      this.logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`);
       return await this.commit(lsn);
     }
 
+    await this.db.write_checkpoints.updateMany(
+      {
+        processed_at_lsn: null,
+        'lsns.1': { $lte: lsn }
+      },
+      {
+        $set: {
+          processed_at_lsn: lsn
+        }
+      },
+      {
+        session: this.session
+      }
+    );
+
     await this.db.sync_rules.updateOne(
       {
         _id: this.group_id
@@ -709,15 +777,33 @@ export class MongoBucketBatch
           snapshot_done: true,
           last_fatal_error: null,
           last_keepalive_ts: new Date()
-        }
+        },
+        $unset: { snapshot_lsn: 1 }
       },
       { session: this.session }
     );
+    await this.db.notifyCheckpoint();
     this.last_checkpoint_lsn = lsn;
 
     return true;
   }
 
+  async setSnapshotLsn(lsn: string): Promise<void> {
+    const update: Partial<SyncRuleDocument> = {
+      snapshot_lsn: lsn
+    };
+
+    await this.db.sync_rules.updateOne(
+      {
+        _id: this.group_id
+      },
+      {
+        $set: update
+      },
+      { session: this.session }
+    );
+  }
+
   async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
     const { after, before, sourceTable, tag } = record;
     for (const event of this.getTableEvents(sourceTable)) {
@@ -742,7 +828,7 @@ export class MongoBucketBatch
       return null;
     }
 
-    logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`);
+    this.logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`);
 
     this.batch ??= new OperationBatch();
     this.batch.push(new RecordOperation(record));
@@ -813,7 +899,7 @@ export class MongoBucketBatch
           session: session
         });
         const batch = await cursor.toArray();
-        const persistedBatch = new PersistedBatch(this.group_id, 0);
+        const persistedBatch = new PersistedBatch(this.group_id, 0, { logger: this.logger });
 
         for (let value of batch) {
           persistedBatch.saveBucketData({
@@ -843,6 +929,37 @@ export class MongoBucketBatch
     return last_op!;
   }
 
+  async updateTableProgress(
+    table: storage.SourceTable,
+    progress: Partial<storage.TableSnapshotStatus>
+  ): Promise<storage.SourceTable> {
+    const copy = table.clone();
+    const snapshotStatus = {
+      totalEstimatedCount: progress.totalEstimatedCount ?? copy.snapshotStatus?.totalEstimatedCount ?? 0,
+      replicatedCount: progress.replicatedCount ?? copy.snapshotStatus?.replicatedCount ?? 0,
+      lastKey: progress.lastKey ?? copy.snapshotStatus?.lastKey ?? null
+    };
+    copy.snapshotStatus = snapshotStatus;
+
+    await this.withTransaction(async () => {
+      await this.db.source_tables.updateOne(
+        { _id: table.id },
+        {
+          $set: {
+            snapshot_status: {
+              last_key: snapshotStatus.lastKey == null ? null : new bson.Binary(snapshotStatus.lastKey),
+              total_estimated_count: snapshotStatus.totalEstimatedCount,
+              replicated_count: snapshotStatus.replicatedCount
+            }
+          }
+        },
+        { session: this.session }
+      );
+    });
+
+    return copy;
+  }
+
   async markSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn: string) {
     const session = this.session;
     const ids = tables.map((table) => table.id);
@@ -853,6 +970,9 @@ export class MongoBucketBatch
         {
           $set: {
             snapshot_done: true
+          },
+          $unset: {
+            snapshot_status: 1
           }
         },
         { session }
@@ -876,17 +996,8 @@ export class MongoBucketBatch
       }
     });
     return tables.map((table) => {
-      const copy = new storage.SourceTable(
-        table.id,
-        table.connectionTag,
-        table.objectId,
-        table.schema,
-        table.table,
-        table.replicaIdColumns,
-        table.snapshotComplete
-      );
-      copy.syncData = table.syncData;
-      copy.syncParameters = table.syncParameters;
+      const copy = table.clone();
+      copy.snapshotComplete = true;
       return copy;
     });
   }
@@ -895,7 +1006,7 @@ export class MongoBucketBatch
    * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
    */
   protected getTableEvents(table: storage.SourceTable): SqlEventDescriptor[] {
-    return this.sync_rules.event_descriptors.filter((evt) =>
+    return this.sync_rules.eventDescriptors.filter((evt) =>
       [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table))
     );
   }

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -15,6 +15,7 @@ export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRule
   public readonly last_fatal_error: string | null;
   public readonly last_keepalive_ts: Date | null;
   public readonly last_checkpoint_ts: Date | null;
+  public readonly active: boolean;
 
   public current_lock: MongoSyncRulesLock | null = null;
 
@@ -30,6 +31,7 @@ export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRule
     this.last_fatal_error = doc.last_fatal_error;
     this.last_checkpoint_ts = doc.last_checkpoint_ts;
     this.last_keepalive_ts = doc.last_keepalive_ts;
+    this.active = doc.state == 'ACTIVE';
   }
 
   parsed(options: storage.ParseSyncRulesOptions) {

package/src/storage/implementation/MongoStorageProvider.ts

@@ -1,5 +1,5 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { MongoStorageConfig } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
@@ -26,6 +26,15 @@ export class MongoStorageProvider implements storage.BucketStorageProvider {
       maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
     });
 
+    let shuttingDown = false;
+
+    // Explicitly connect on startup.
+    // Connection errors during startup are typically not recoverable - we get topologyClosed.
+    // This helps to catch the error early, along with the cause, and before the process starts
+    // to serve API requests.
+    // Errors here will cause the process to exit.
+    await client.connect();
+
     const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
     const factory = new MongoBucketStorage(database, {
       // TODO currently need the entire resolved config due to this
@@ -34,12 +43,29 @@ export class MongoStorageProvider implements storage.BucketStorageProvider {
     return {
       storage: factory,
       shutDown: async () => {
+        shuttingDown = true;
         await factory[Symbol.asyncDispose]();
         await client.close();
       },
       tearDown: () => {
         logger.info(`Tearing down storage: ${database.db.namespace}...`);
         return database.db.dropDatabase();
+      },
+      onFatalError: (callback) => {
+        client.addListener('topologyClosed', () => {
+          // If we're shutting down, this is expected and we can ignore it.
+          if (!shuttingDown) {
+            // Unfortunately there is no simple way to catch the cause of this issue.
+            // It most commonly happens when the process fails to _ever_ connect - connection issues after
+            // the initial connection are usually recoverable.
+            callback(
+              new ServiceError({
+                code: ErrorCode.PSYNC_S2402,
+                description: 'MongoDB topology closed - failed to connect to MongoDB storage.'
+              })
+            );
+          }
+        });
       }
     } satisfies storage.ActiveStorage;
   }