@powersync/service-module-mongodb-storage 0.21.0 → 0.22.0

package/src/storage/implementation/common/PersistedBatch.ts

@@ -8,6 +8,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
 import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
 import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
 import { MongoIdSequence } from '../MongoIdSequence.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import type { VersionedPowerSyncMongo } from '../db.js';
 import { TaggedBucketParameterDocument } from '../models.js';
 import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
@@ -216,13 +217,13 @@ export abstract class PersistedBatch {
 
   protected abstract get currentDataCount(): number;
 
-  protected abstract flushBucketData(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
 
-  protected abstract flushBucketParameters(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
 
-  protected abstract flushCurrentData(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueCurrentData(writes: MongoWriteBatch): void;
 
-  protected abstract flushBucketStates(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueBucketStates(writes: MongoWriteBatch): void;
 
   protected abstract resetCurrentData(): void;
 
@@ -348,24 +349,27 @@ export abstract class PersistedBatch {
   async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
     const startAt = performance.now();
     let flushedSomething = false;
+    const writes = this.db.createWriteBatch(session, { ordered: false });
     if (this.bucketDataCount > 0) {
       flushedSomething = true;
-      await this.flushBucketData(session);
+      await this.queueBucketData(writes);
     }
     if (this.bucketParameters.length > 0) {
       flushedSomething = true;
-      await this.flushBucketParameters(session);
+      this.queueBucketParameters(writes);
     }
     if (this.currentDataCount > 0) {
       flushedSomething = true;
-      await this.flushCurrentData(session);
+      this.queueCurrentData(writes);
     }
 
     if (this.bucketStates.size > 0) {
       flushedSomething = true;
-      await this.flushBucketStates(session);
+      this.queueBucketStates(writes);
     }
 
+    await writes.execute();
+
     if (flushedSomething) {
       const duration = Math.round(performance.now() - startAt);
       if (options?.oldestUncommittedChange != null) {

package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts

@@ -58,6 +58,10 @@ export abstract class BaseVersionedPowerSyncMongo {
     return this.upstream.notifyCheckpoint();
   }
 
+  createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }) {
+    return this.upstream.createWriteBatch(session, options);
+  }
+
   protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
     return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
   }

package/src/storage/implementation/db.ts

@@ -17,6 +17,7 @@ import {
   SyncRuleDocumentBase,
   WriteCheckpointDocument
 } from './models.js';
+import { MongoWriteBatch } from './MongoWriteBatch.js';
 import {
   BucketDataDocumentV1,
   BucketParameterDocument,
@@ -54,6 +55,19 @@ export class PowerSyncMongo {
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
+  private get clientBulkWriteSupport(): boolean {
+    // The driver exposes topology at runtime, but omits this internal property
+    // from its public types. Reuse its handshake results without another command.
+    const client = this.client as mongo.MongoClient & { topology?: { description: mongo.TopologyDescription } };
+    const servers = [...(client.topology?.description.servers.values() ?? [])];
+    // MongoDB 8.0 introduced client bulkWrite at wire version 25. Unknown / mixed-version
+    // topologies use collection writes until all known servers support it.
+    return servers.length > 0 && servers.every((server) => server.maxWireVersion >= 25);
+  }
+
+  createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }): MongoWriteBatch {
+    return new MongoWriteBatch(this.client, this.clientBulkWriteSupport, session, options);
+  }
 
   constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions) {
     this.client = client;

package/src/storage/implementation/v1/MongoBucketBatchV1.ts

@@ -55,8 +55,15 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     session: mongo.ClientSession,
     opId: InternalOpId
   ): Promise<void> {
+    // Only the final checkpoint per user/stream is visible after this transaction.
+    const checkpoints = new Map(
+      this.write_checkpoint_batch.map((checkpoint) => [
+        JSON.stringify([checkpoint.sync_rules_id, checkpoint.user_id]),
+        checkpoint
+      ])
+    );
     await this.db.custom_write_checkpoints.bulkWrite(
-      this.write_checkpoint_batch.map((checkpointOptions) => {
+      [...checkpoints.values()].map((checkpointOptions) => {
         const set: Record<string, unknown> = {
           checkpoint: checkpointOptions.checkpoint,
           sync_rules_id: checkpointOptions.sync_rules_id,
@@ -77,7 +84,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
           }
         };
       }),
-      { session }
+      { session, ordered: false }
     );
   }
 
@@ -462,7 +469,9 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     const ids = tables.map((table) => mongoTableId(table.id));
 
     await this.withTransaction(async () => {
-      await this.db.sourceTablesV1(this.replicationStreamId).updateMany(
+      const writes = this.db.createWriteBatch(session, { ordered: false });
+      writes.updateMany(
+        this.db.sourceTablesV1(this.replicationStreamId),
         { _id: { $in: ids } },
         {
           $set: {
@@ -471,12 +480,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
           $unset: {
             snapshot_status: 1
           }
-        },
-        { session }
+        }
       );
 
       if (no_checkpoint_before_lsn != null) {
-        await this.db.sync_rules.updateOne(
+        writes.updateOne(
+          this.db.sync_rules,
           {
             _id: this.replicationStreamId
           },
@@ -487,10 +496,10 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
             $max: {
               no_checkpoint_before: no_checkpoint_before_lsn
             }
-          },
-          { session: this.session }
+          }
         );
       }
+      await writes.execute();
     });
     return tables.map((table) => {
       const copy = table.clone();

package/src/storage/implementation/v1/MongoCompactorV1.ts

@@ -13,6 +13,7 @@ import { BucketDefinitionId } from '@powersync/service-sync-rules';
 import { BucketDataDoc } from '../common/BucketDataDoc.js';
 import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
 import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import { cacheKey } from '../OperationBatch.js';
 import { BucketDataDocumentV1, BucketStateDocumentV1 } from './models.js';
 import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js';
@@ -296,11 +297,10 @@ export class MongoCompactorV1 extends MongoCompactor {
   }
 
   private async flushBucketStateUpdates() {
-    if (this.bucketStateUpdates.length > 0) {
-      this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
-      await this.writeBucketStateUpdates();
-      this.bucketStateUpdates = [];
-    }
+    const writes = this.db.createWriteBatch(undefined, { ordered: false });
+    this.writeBucketStateUpdates(writes);
+    await writes.execute();
+    this.bucketStateUpdates = [];
   }
 
   private async updateChecksumsBatch(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]) {
@@ -340,10 +340,12 @@ export class MongoCompactorV1 extends MongoCompactor {
     await this.flushBucketStateUpdates();
   }
 
-  protected async writeBucketStateUpdates(): Promise<void> {
-    await this.db.bucketStateV1.bulkWrite(
-      this.bucketStateUpdates as mongo.AnyBulkWriteOperation<BucketStateDocumentV1>[],
-      { ordered: false }
+  protected writeBucketStateUpdates(writes: MongoWriteBatch): void {
+    if (this.bucketStateUpdates.length === 0) return;
+    this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
+    writes.bulkWriteUnordered(
+      this.db.bucketStateV1,
+      this.bucketStateUpdates as mongo.AnyBulkWriteOperation<BucketStateDocumentV1>[]
     );
   }
 
@@ -541,17 +543,19 @@ export class MongoCompactorV1 extends MongoCompactor {
   }
 
   private async flush(bucketContext: SingleBucketStoreV1) {
+    // Persist data changes before advertising the resulting bucket state.
+    const writes = this.db.createWriteBatch(undefined, { ordered: true });
     if (this.updates.length > 0) {
       this.logger.info(`Compacting ${this.updates.length} ops`);
-      await bucketContext.collection.bulkWrite(this.updates, {
-        // Order is not important. Since checksums are not affected, these operations can happen in any order,
-        // and it's fine if the operations are partially applied. Each individual operation is atomic.
-        ordered: false
-      });
-      this.updates = [];
+      // Order is not important. Since checksums are not affected, these operations can happen in any order,
+      // and it's fine if the operations are partially applied. Each individual operation is atomic.
+      writes.bulkWriteUnordered(bucketContext.collection, this.updates);
     }
 
-    await this.flushBucketStateUpdates();
+    this.writeBucketStateUpdates(writes);
+    await writes.execute();
+    this.updates = [];
+    this.bucketStateUpdates = [];
   }
 
   /**
@@ -622,15 +626,13 @@ export class MongoCompactorV1 extends MongoCompactor {
             }
 
             this.logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`);
-            await bucketContext.collection.deleteMany(
-              {
-                _id: {
-                  $gte: bucketContext.minId,
-                  $lte: bucketContext.docId(lastOp!.o)
-                }
-              },
-              { session }
-            );
+            const writes = this.db.createWriteBatch(session, { ordered: true });
+            writes.deleteMany(bucketContext.collection, {
+              _id: {
+                $gte: bucketContext.minId,
+                $lte: bucketContext.docId(lastOp!.o)
+              }
+            });
 
             const op = bucketContext.toPersistedDocument({
               o: lastOp!.o,
@@ -639,7 +641,8 @@ export class MongoCompactorV1 extends MongoCompactor {
               data: null,
               target_op: targetOp
             });
-            await bucketContext.collection.insertOne(op, { session });
+            writes.insertOne(bucketContext.collection, op);
+            await writes.execute();
 
             opCountDiff = -numberOfOpsToClear + 1;
           },

package/src/storage/implementation/v1/PersistedBatchV1.ts

@@ -6,6 +6,7 @@ import * as bson from 'bson';
 import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
 import { mongoTableId } from '../../../utils/util.js';
 import { EMPTY_DATA } from '../MongoBucketBatchShared.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import {
   BucketStateUpdate,
   PersistedBatch,
@@ -25,7 +26,9 @@ import {
 export class PersistedBatchV1 extends PersistedBatch {
   declare protected readonly db: VersionedPowerSyncMongoV1;
 
-  currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[] = [];
+  // Each upsert supplies the complete source-record state. Only its final
+  // state matters within a flush; bucket and parameter history is kept separately.
+  currentData = new Map<string, mongo.AnyBulkWriteOperation<CurrentDataDocument>>();
 
   protected checkDefinitionId(_definitionId: BucketDefinitionId | null): BucketDefinitionId {
     // V1 storage doesn't persist the id, and we don't use it.
@@ -94,7 +97,7 @@ export class PersistedBatchV1 extends PersistedBatch {
   }
 
   hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) {
-    this.currentData.push({
+    this.currentData.set(this.currentDataKey(sourceTableId, replicaId), {
       deleteOne: {
         filter: { _id: this.currentDataId(sourceTableId, replicaId) }
       }
@@ -124,7 +127,7 @@ export class PersistedBatchV1 extends PersistedBatch {
       return lookup.lookup;
     });
 
-    this.currentData.push({
+    this.currentData.set(this.currentDataKey(values.sourceTableId, values.replicaId), {
       updateOne: {
         filter: { _id: this.currentDataId(values.sourceTableId, values.replicaId) },
         update: {
@@ -141,57 +144,45 @@ export class PersistedBatchV1 extends PersistedBatch {
   }
 
   protected get currentDataCount() {
-    return this.currentData.length;
+    return this.currentData.size;
   }
 
-  protected async flushBucketData(session: mongo.ClientSession) {
-    await this.db.bucketDataV1.bulkWrite(
+  protected async queueBucketData(writes: MongoWriteBatch) {
+    writes.bulkWriteUnordered(
+      this.db.bucketDataV1,
       this.bucketData.map((document) => ({
         insertOne: {
           document: serializeBucketDataV1(document)
         }
-      })),
-      {
-        session,
-        ordered: false
-      }
+      }))
     );
   }
 
-  protected async flushBucketParameters(session: mongo.ClientSession) {
-    await this.db.parameterIndexV1.bulkWrite(
+  protected queueBucketParameters(writes: MongoWriteBatch): void {
+    writes.bulkWriteUnordered(
+      this.db.parameterIndexV1,
       this.bucketParameters.map((document) => ({
         insertOne: {
           document: taggedBucketParameterDocumentToV1(document)
         }
-      })),
-      {
-        session,
-        ordered: false
-      }
+      }))
     );
   }
 
-  protected async flushCurrentData(session: mongo.ClientSession) {
-    if (this.currentData.length == 0) {
+  protected queueCurrentData(writes: MongoWriteBatch): void {
+    if (this.currentData.size == 0) {
       return;
     }
 
-    await this.db.sourceRecordsV1.bulkWrite(this.currentData, {
-      session,
-      ordered: true
-    });
+    writes.bulkWriteUnordered(this.db.sourceRecordsV1, [...this.currentData.values()]);
   }
 
-  protected async flushBucketStates(session: mongo.ClientSession) {
-    await this.db.bucketStateV1.bulkWrite(this.getBucketStateUpdates(), {
-      session,
-      ordered: false
-    });
+  protected queueBucketStates(writes: MongoWriteBatch): void {
+    writes.bulkWriteUnordered(this.db.bucketStateV1, this.getBucketStateUpdates());
   }
 
   protected resetCurrentData() {
-    this.currentData = [];
+    this.currentData.clear();
   }
 
   private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocumentV1>[] {
@@ -219,6 +210,10 @@ export class PersistedBatchV1 extends PersistedBatch {
     });
   }
 
+  private currentDataKey(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): string {
+    return Buffer.from(bson.serialize({ t: sourceTableId, k: replicaId })).toString('base64');
+  }
+
   private currentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey {
     return {
       g: this.group_id,

package/src/storage/implementation/v3/MongoBucketBatchV3.ts

@@ -7,6 +7,7 @@ import { mongoTableId } from '../../../utils/util.js';
 import { canCheckpointState } from '../CheckpointState.js';
 import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
 import { MongoParsedSyncConfigSet } from '../MongoParsedSyncConfigSet.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import { stopReplicationStreamPipeline } from '../SyncRuleStateUpdate.js';
 import { PersistedBatch } from '../common/PersistedBatch.js';
 import { SourceRecordStore } from '../common/SourceRecordStore.js';
@@ -60,14 +61,12 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
     return this.store;
   }
 
-  protected override async onReplicationTransactionFlush(
-    session: lib_mongo.mongo.ClientSession,
-    lastOp: bigint
-  ): Promise<void> {
+  protected override onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: bigint): void {
     // Durably advance the stream-level head of persisted ops within the flush transaction.
     // This ensures a checkpoint created later (even by an empty commit, or by a freshly-appended
     // config that replicates nothing) covers all ops persisted before a potential crash.
-    await this.db.sync_rules.updateOne(
+    writes.updateOne(
+      this.db.sync_rules,
       {
         _id: this.replicationStreamId
       },
@@ -75,8 +74,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
         $max: {
           last_persisted_op: lastOp
         }
-      },
-      { session }
+      }
     );
   }
 
@@ -652,7 +650,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
     const syncConfigIds = this.relevantSyncConfigIdsForTables(tables);
 
     await this.withTransaction(async () => {
-      await this.db.sourceTables(this.replicationStreamId).updateMany(
+      const writes = this.db.createWriteBatch(session, { ordered: false });
+      writes.updateMany(
+        this.db.sourceTables(this.replicationStreamId),
         { _id: { $in: ids } },
         {
           $set: {
@@ -661,12 +661,12 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
           $unset: {
             snapshot_status: 1
           }
-        },
-        { session }
+        }
       );
 
       if (no_checkpoint_before_lsn != null && syncConfigIds.length > 0) {
-        await this.db.sync_rules.updateOne(
+        writes.updateOne(
+          this.db.sync_rules,
           {
             _id: this.replicationStreamId,
             'sync_configs._id': { $in: syncConfigIds }
@@ -680,12 +680,12 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
             }
           },
           {
-            session: this.session,
             // Only set for sync configs that use this table
             arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
           }
         );
       }
+      await writes.execute();
     });
     return tables.map((table) => {
       const copy = table.clone();
@@ -706,9 +706,11 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
       this.validateCustomCheckpointEventId(checkpoint.event_id)
     );
 
+    const writes = this.db.createWriteBatch(session, { ordered: false });
     for (const [eventId, checkpoints] of checkpointsByEvent) {
-      await this.batchCreateEventCustomWriteCheckpoints(session, opId, eventId, checkpoints);
+      this.batchCreateEventCustomWriteCheckpoints(writes, opId, eventId, checkpoints);
     }
+    await writes.execute();
   }
 
   protected override async prepareCustomWriteCheckpoints(): Promise<void> {
@@ -741,48 +743,48 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
     return eventId;
   }
 
-  private async batchCreateEventCustomWriteCheckpoints(
-    session: lib_mongo.mongo.ClientSession,
+  private batchCreateEventCustomWriteCheckpoints(
+    writes: MongoWriteBatch,
     opId: InternalOpId,
     eventId: EventDefinitionId,
     checkpoints: storage.CustomWriteCheckpointOptions[]
-  ): Promise<void> {
-    await this.db
-      .customCheckpointRequests({
+  ): void {
+    // A repeated user within an event replaces the complete checkpoint state.
+    const uniqueCheckpoints = new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint]));
+    writes.bulkWriteUnordered(
+      this.db.customCheckpointRequests({
         eventId,
         replicationStreamId: this.replicationStreamId
-      })
-      .bulkWrite(
-        checkpoints.map((checkpointOptions) => {
-          const set: Partial<CustomCheckpointRequestDocumentV3> = {
-            user_id: checkpointOptions.user_id,
-            checkpoint: checkpointOptions.checkpoint,
-            op_id: opId
-          };
-          if (checkpointOptions.checkpoint_requested_at != null) {
-            set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
-          }
+      }),
+      [...uniqueCheckpoints.values()].map((checkpointOptions) => {
+        const set: Partial<CustomCheckpointRequestDocumentV3> = {
+          user_id: checkpointOptions.user_id,
+          checkpoint: checkpointOptions.checkpoint,
+          op_id: opId
+        };
+        if (checkpointOptions.checkpoint_requested_at != null) {
+          set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
+        }
 
-          return {
-            updateOne: {
-              filter: {
-                user_id: checkpointOptions.user_id
-              },
-              update: {
-                $set: set,
-                ...(checkpointOptions.checkpoint_requested_at == null
-                  ? {
-                      $unset: {
-                        checkpoint_requested_at: 1
-                      }
+        return {
+          updateOne: {
+            filter: {
+              user_id: checkpointOptions.user_id
+            },
+            update: {
+              $set: set,
+              ...(checkpointOptions.checkpoint_requested_at == null
+                ? {
+                    $unset: {
+                      checkpoint_requested_at: 1
                     }
-                  : {})
-              },
-              upsert: true
-            }
-          };
-        }),
-        { session }
-      );
+                  }
+                : {})
+            },
+            upsert: true
+          }
+        };
+      })
+    );
   }
 }