@powersync/service-module-mongodb-storage 0.22.0 → 0.23.0

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -1,8 +1,8 @@
 import crypto from 'crypto';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
+import { ErrorCode, Logger, ReplicationAbortedError, ServiceError } from '@powersync/lib-services-framework';
+import { storage, utils } from '@powersync/service-core';
 import { VersionedPowerSyncMongo } from './db.js';
 
 const LOCK_DURATION_MS = 60 * 1000;
@@ -12,6 +12,12 @@ const LOCK_DURATION_MS = 60 * 1000;
  * processes that replication stream at a time.
  */
 export class MongoSyncRulesLock implements storage.ReplicationLock {
+  /** FIFO admission shared by this lease's writers; database fencing still enforces ownership. */
+  readonly writerMutex = new utils.Mutex();
+
+  private readonly abort = new AbortController();
+  readonly signal = this.abort.signal;
+
   private readonly refreshInterval: NodeJS.Timeout;
 
   /**
@@ -67,13 +73,14 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   constructor(
     private db: VersionedPowerSyncMongo,
     public sync_rules_id: number,
-    private lock_id: string,
+    public readonly lock_id: string,
     private logger: Logger
   ) {
     this.refreshInterval = setInterval(async () => {
       try {
         await this.refresh();
       } catch (e) {
+        this.abort.abort(e);
         this.logger.error('Failed to refresh lock', e);
         clearInterval(this.refreshInterval);
       }
@@ -81,6 +88,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   }
 
   async release(): Promise<void> {
+    this.abort.abort(new Error('Replication lock released'));
     clearInterval(this.refreshInterval);
     const result = await this.db.sync_rules.updateOne(
       {
@@ -97,6 +105,51 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
     }
   }
 
+  throwIfAborted(): void {
+    this.signal.throwIfAborted();
+  }
+
+  static ownerFilter(streamId: number, lock: MongoSyncRulesLock) {
+    lock.throwIfAborted();
+    return { _id: streamId, 'lock.id': lock.lock_id };
+  }
+
+  static heartbeatUpdate() {
+    // Always change the existing heartbeat, even for writes in the same
+    // millisecond. A no-op update is not sufficient for a transactional fence.
+    return {
+      last_keepalive_ts: {
+        $max: ['$$NOW', { $add: [{ $ifNull: ['$last_keepalive_ts', new Date(0)] }, 1] }]
+      }
+    };
+  }
+
+  static assertOwned<T>(document: T | null): T {
+    if (document == null) {
+      throw new ReplicationAbortedError('Replication writer no longer owns the stream');
+    }
+    return document;
+  }
+
+  /**
+   * A write, not just an ownership read: takeover conflicts with every transaction
+   * that publishes under this owner. Every writer must hold a stream lease.
+   */
+  static async fence(
+    db: VersionedPowerSyncMongo,
+    streamId: number,
+    lock: MongoSyncRulesLock,
+    session: mongo.ClientSession,
+    projection: mongo.Document = { last_persisted_op: 1, last_checkpoint: 1, keepalive_op: 1 }
+  ) {
+    const doc = await db.sync_rules.findOneAndUpdate(
+      this.ownerFilter(streamId, lock),
+      [{ $set: this.heartbeatUpdate() }],
+      { session, returnDocument: 'after', projection }
+    );
+    return this.assertOwned(doc);
+  }
+
   private async refresh(): Promise<void> {
     const result = await this.db.sync_rules.findOneAndUpdate(
       {

package/src/storage/implementation/db.ts

@@ -81,6 +81,7 @@ export class PowerSyncMongo {
     this.bucket_data = db.collection('bucket_data');
     this.bucket_parameters = db.collection('bucket_parameters');
     this.op_id_sequence = db.collection('op_id_sequence');
+    // Uses legacy terminology instead of "sync config" for backwards-compatibility
     this.sync_rules = db.collection('sync_rules');
     this.source_tables = db.collection('source_tables');
     this.custom_write_checkpoints = db.collection('custom_write_checkpoints');

package/src/storage/implementation/models.ts

@@ -308,6 +308,7 @@ export interface CustomWriteCheckpointDocument {
   _id: bson.ObjectId;
   user_id: string;
   checkpoint: bigint;
+  // Uses the legacy field name "sync_rules_id" for backwards-compatibility. See AGENTS.md.
   sync_rules_id: number;
   /**
    * Unlike managed write checkpoints, custom write checkpoints are flushed together with

package/src/storage/implementation/v1/MongoBucketBatchV1.ts

@@ -6,8 +6,11 @@ import * as bson from 'bson';
 import { mongoTableId } from '../../../utils/util.js';
 import { calculateCheckpointState } from '../CheckpointState.js';
 import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { MongoSyncRulesLock } from '../MongoSyncRulesLock.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import { PersistedBatch } from '../common/PersistedBatch.js';
 import { SourceRecordStore } from '../common/SourceRecordStore.js';
+import { SyncRuleDocumentBase } from '../models.js';
 import { PersistedBatchV1 } from './PersistedBatchV1.js';
 import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
@@ -34,6 +37,27 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId);
   }
 
+  protected override persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId {
+    const legacy = stream as SyncRuleDocumentV1;
+    const checkpoint = legacy.last_checkpoint ?? 0n;
+    const keepalive = legacy.keepalive_op == null ? 0n : BigInt(legacy.keepalive_op);
+    return checkpoint > keepalive ? checkpoint : keepalive;
+  }
+
+  protected override onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void {
+    // Keep flushed operations recoverable before a checkpoint can be published.
+    // keepalive_op remains a decimal string for compatibility with legacy readers.
+    writes.updateOne(this.db.sync_rules, { _id: this.replicationStreamId }, [
+      {
+        $set: {
+          keepalive_op: {
+            $toString: { $max: [{ $toLong: '$keepalive_op' }, '$last_checkpoint', lastOp] }
+          }
+        }
+      }
+    ]);
+  }
+
   protected override recordPersistedOp(lastOp: InternalOpId): void {
     this.persisted_op = lastOp;
   }
@@ -101,7 +125,8 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     }));
 
     let result: storage.ResolveTablesResult | null = null;
-    await this.db.client.withSession(async (session) => {
+    const session = this.session;
+    await this.withTransaction(async () => {
       const col = this.db.sourceTablesV1(this.replicationStreamId);
 
       // Find records that overlap by name or relation id.
@@ -219,129 +244,136 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
   async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult> {
     const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
 
-    await this.flush(options);
-
     using _ = this.tracer.span('storage', 'commit');
 
-    const now = new Date();
+    const updateCheckpoint = async (persistedOp = this.persisted_op) => {
+      const now = new Date();
 
-    await this.db.write_checkpoints.updateMany(
-      {
-        processed_at_lsn: null,
-        'lsns.1': { $lte: lsn }
-      },
-      {
-        $set: {
-          processed_at_lsn: lsn
-        }
-      },
-      {
-        session: this.session
-      }
-    );
+      const can_checkpoint = {
+        $and: [
+          { $eq: ['$snapshot_done', true] },
+          {
+            $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
+          },
+          {
+            $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
+          }
+        ]
+      };
 
-    const can_checkpoint = {
-      $and: [
-        { $eq: ['$snapshot_done', true] },
-        {
-          $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
-        },
-        {
-          $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
-        }
-      ]
-    };
+      const new_keepalive_op = {
+        $cond: [
+          can_checkpoint,
+          { $literal: null },
+          {
+            $toString: {
+              $max: [{ $toLong: '$keepalive_op' }, { $literal: persistedOp }, 0n]
+            }
+          }
+        ]
+      };
+
+      const new_last_checkpoint = {
+        $cond: [
+          can_checkpoint,
+          {
+            $max: ['$last_checkpoint', { $literal: persistedOp }, { $toLong: '$keepalive_op' }, 0n]
+          },
+          '$last_checkpoint'
+        ]
+      };
 
-    const new_keepalive_op = {
-      $cond: [
-        can_checkpoint,
-        { $literal: null },
+      const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(
+        MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock),
+        [
+          {
+            $set: {
+              _can_checkpoint: can_checkpoint,
+              _not_empty: createEmptyCheckpoints
+                ? true
+                : {
+                    $or: [
+                      { $literal: createEmptyCheckpoints },
+                      { $ne: ['$keepalive_op', new_keepalive_op] },
+                      { $ne: ['$last_checkpoint', new_last_checkpoint] }
+                    ]
+                  }
+            }
+          },
+          {
+            $set: {
+              ...MongoSyncRulesLock.heartbeatUpdate(),
+              last_checkpoint_lsn: {
+                $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
+              },
+              last_checkpoint_ts: {
+                $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
+              },
+
+              last_fatal_error: { $literal: null },
+              last_fatal_error_ts: { $literal: null },
+              keepalive_op: new_keepalive_op,
+              last_checkpoint: new_last_checkpoint,
+              snapshot_lsn: {
+                $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
+              }
+            }
+          },
+          {
+            $unset: ['_can_checkpoint', '_not_empty']
+          }
+        ],
         {
-          $toString: {
-            $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
+          session: this.session,
+          returnDocument: 'before',
+          projection: {
+            snapshot_done: 1,
+            last_checkpoint_lsn: 1,
+            no_checkpoint_before: 1,
+            keepalive_op: 1,
+            last_checkpoint: 1
           }
         }
-      ]
-    };
+      )) as SyncRuleDocumentV1;
 
-    const new_last_checkpoint = {
-      $cond: [
-        can_checkpoint,
-        {
-          $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
-        },
-        '$last_checkpoint'
-      ]
+      return MongoSyncRulesLock.assertOwned(preUpdateDocument);
     };
 
-    const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(
-      { _id: this.replicationStreamId },
-      [
+    const finishCheckpoint = async (preUpdateDocument: SyncRuleDocumentV1, lastOp?: InternalOpId) => {
+      await this.db.write_checkpoints.updateMany(
         {
-          $set: {
-            _can_checkpoint: can_checkpoint,
-            _not_empty: createEmptyCheckpoints
-              ? true
-              : {
-                  $or: [
-                    { $literal: createEmptyCheckpoints },
-                    { $ne: ['$keepalive_op', new_keepalive_op] },
-                    { $ne: ['$last_checkpoint', new_last_checkpoint] }
-                  ]
-                }
-          }
+          processed_at_lsn: null,
+          'lsns.1': { $lte: lsn }
         },
         {
           $set: {
-            last_checkpoint_lsn: {
-              $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
-            },
-            last_checkpoint_ts: {
-              $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
-            },
-            last_keepalive_ts: { $literal: now },
-            last_fatal_error: { $literal: null },
-            last_fatal_error_ts: { $literal: null },
-            keepalive_op: new_keepalive_op,
-            last_checkpoint: new_last_checkpoint,
-            snapshot_lsn: {
-              $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
-            }
+            processed_at_lsn: lsn
           }
         },
         {
-          $unset: ['_can_checkpoint', '_not_empty']
+          session: this.session
         }
-      ],
-      {
-        session: this.session,
-        returnDocument: 'before',
-        projection: {
-          snapshot_done: 1,
-          last_checkpoint_lsn: 1,
-          no_checkpoint_before: 1,
-          keepalive_op: 1,
-          last_checkpoint: 1
-        }
-      }
-    )) as SyncRuleDocumentV1;
-
-    if (preUpdateDocument == null) {
-      throw new ReplicationAssertionError(
-        'Failed to update checkpoint - no matching sync_rules document for _id: ' + this.replicationStreamId
       );
-    }
 
-    const checkpointState = calculateCheckpointState({
-      lsn,
-      snapshotDone: preUpdateDocument.snapshot_done === true,
-      lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
-      noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
-      keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
-      lastCheckpoint: preUpdateDocument.last_checkpoint,
-      persistedOp: this.persisted_op,
-      createEmptyCheckpoints
-    });
+      const checkpointState = calculateCheckpointState({
+        lsn,
+        snapshotDone: preUpdateDocument.snapshot_done === true,
+        lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
+        noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
+        keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
+        lastCheckpoint: preUpdateDocument.last_checkpoint,
+        persistedOp: lastOp ?? this.persistedOpHead(preUpdateDocument),
+        createEmptyCheckpoints
+      });
+      return { checkpointState, preUpdateDocument };
+    };
+
+    const { checkpointState, preUpdateDocument } = await this.flushAndCommit(
+      async (_stream, lastOp) => finishCheckpoint(await updateCheckpoint(lastOp), lastOp),
+      // Without a flush, the checkpoint update itself fences the transaction.
+      () => this.withFencedTransaction(updateCheckpoint, finishCheckpoint),
+      options
+    );
     if (checkpointState.checkpointBlocked) {
       if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
         this.logger.info(
@@ -379,41 +411,20 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
 
   async setResumeLsn(lsn: string): Promise<void> {
     using _ = this.tracer.span('storage', 'set_resume_lsn');
-    await this.db.sync_rules.updateOne(
-      {
-        _id: this.replicationStreamId
-      },
-      {
-        $set: {
-          snapshot_lsn: lsn
-        }
-      },
-      {
-        session: this.session,
-        // Losing occasional resume LSN is fine. That may mean reprocessing
-        // some source changes in some edge cases, which is not an issue since
-        // changes are processed in an idempotent way.
-        writeConcern: { w: 1 }
-      }
-    );
+    // Losing occasional resume LSN would only reprocess source changes.
+    // Keep the lease check atomic, but do not wait for majority replication of this resume hint.
+    await this.updateStreamMetadata({ snapshot_lsn: { $literal: lsn } }, {}, { w: 1 });
   }
 
   async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void> {
-    await this.db.sync_rules.updateOne(
-      {
-        _id: this.replicationStreamId
-      },
-      {
-        $set: {
-          snapshot_done: true,
-          last_keepalive_ts: new Date()
-        },
-        $max: {
-          no_checkpoint_before: no_checkpoint_before_lsn
-        }
-      },
-      { session: this.session }
-    );
+    await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
+  }
+
+  private snapshotDoneUpdate(no_checkpoint_before_lsn: string) {
+    return {
+      snapshot_done: true,
+      no_checkpoint_before: { $max: ['$no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
+    };
   }
 
   async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
@@ -443,22 +454,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
         }
       }
 
-      await this.markAllSnapshotDone(no_checkpoint_before_lsn);
+      await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
     });
   }
 
   async markTableSnapshotRequired(_table: storage.SourceTable): Promise<void> {
-    await this.db.sync_rules.updateOne(
-      {
-        _id: this.replicationStreamId
-      },
-      {
-        $set: {
-          snapshot_done: false
-        }
-      },
-      { session: this.session }
-    );
+    await this.updateStreamMetadata({ snapshot_done: false });
   }
 
   async markTableSnapshotDone(
@@ -490,9 +491,6 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
             _id: this.replicationStreamId
           },
           {
-            $set: {
-              last_keepalive_ts: new Date()
-            },
             $max: {
               no_checkpoint_before: no_checkpoint_before_lsn
             }
@@ -516,7 +514,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     const session = this.session;
     let activated = false;
     let needsFutureActivationCheck = true;
-    await session.withTransaction(async () => {
+    await this.withTransaction(async () => {
       // Reset on transaction retries.
       activated = false;
       needsFutureActivationCheck = true;
@@ -554,7 +552,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
       } else if (doc?.state != storage.SyncRuleState.PROCESSING) {
         needsFutureActivationCheck = false;
       }
-    });
+    }, session);
     if (activated) {
       this.logger.info(`Activated new replication stream at ${lsn}`);
       await this.db.notifyCheckpoint();

package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts

@@ -90,7 +90,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     if (keepaliveOp == null && lastCheckpoint == null) {
       return null;
     }
-    return (keepaliveOp ?? 0n) > (lastCheckpoint ?? 0n) ? keepaliveOp : lastCheckpoint;
+    return [keepaliveOp ?? 0n, lastCheckpoint ?? 0n].reduce((a, b) => (a > b ? a : b));
   }
 
   protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {