@powersync/service-module-mongodb-storage 0.21.0 → 0.23.0

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -1,8 +1,8 @@
 import crypto from 'crypto';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
+import { ErrorCode, Logger, ReplicationAbortedError, ServiceError } from '@powersync/lib-services-framework';
+import { storage, utils } from '@powersync/service-core';
 import { VersionedPowerSyncMongo } from './db.js';
 
 const LOCK_DURATION_MS = 60 * 1000;
@@ -12,6 +12,12 @@ const LOCK_DURATION_MS = 60 * 1000;
  * processes that replication stream at a time.
  */
 export class MongoSyncRulesLock implements storage.ReplicationLock {
+  /** FIFO admission shared by this lease's writers; database fencing still enforces ownership. */
+  readonly writerMutex = new utils.Mutex();
+
+  private readonly abort = new AbortController();
+  readonly signal = this.abort.signal;
+
   private readonly refreshInterval: NodeJS.Timeout;
 
   /**
@@ -67,13 +73,14 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   constructor(
     private db: VersionedPowerSyncMongo,
     public sync_rules_id: number,
-    private lock_id: string,
+    public readonly lock_id: string,
     private logger: Logger
   ) {
     this.refreshInterval = setInterval(async () => {
       try {
         await this.refresh();
       } catch (e) {
+        this.abort.abort(e);
         this.logger.error('Failed to refresh lock', e);
         clearInterval(this.refreshInterval);
       }
@@ -81,6 +88,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   }
 
   async release(): Promise<void> {
+    this.abort.abort(new Error('Replication lock released'));
     clearInterval(this.refreshInterval);
     const result = await this.db.sync_rules.updateOne(
       {
@@ -97,6 +105,51 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
     }
   }
 
+  throwIfAborted(): void {
+    this.signal.throwIfAborted();
+  }
+
+  static ownerFilter(streamId: number, lock: MongoSyncRulesLock) {
+    lock.throwIfAborted();
+    return { _id: streamId, 'lock.id': lock.lock_id };
+  }
+
+  static heartbeatUpdate() {
+    // Always change the existing heartbeat, even for writes in the same
+    // millisecond. A no-op update is not sufficient for a transactional fence.
+    return {
+      last_keepalive_ts: {
+        $max: ['$$NOW', { $add: [{ $ifNull: ['$last_keepalive_ts', new Date(0)] }, 1] }]
+      }
+    };
+  }
+
+  static assertOwned<T>(document: T | null): T {
+    if (document == null) {
+      throw new ReplicationAbortedError('Replication writer no longer owns the stream');
+    }
+    return document;
+  }
+
+  /**
+   * A write, not just an ownership read: takeover conflicts with every transaction
+   * that publishes under this owner. Every writer must hold a stream lease.
+   */
+  static async fence(
+    db: VersionedPowerSyncMongo,
+    streamId: number,
+    lock: MongoSyncRulesLock,
+    session: mongo.ClientSession,
+    projection: mongo.Document = { last_persisted_op: 1, last_checkpoint: 1, keepalive_op: 1 }
+  ) {
+    const doc = await db.sync_rules.findOneAndUpdate(
+      this.ownerFilter(streamId, lock),
+      [{ $set: this.heartbeatUpdate() }],
+      { session, returnDocument: 'after', projection }
+    );
+    return this.assertOwned(doc);
+  }
+
   private async refresh(): Promise<void> {
     const result = await this.db.sync_rules.findOneAndUpdate(
       {

package/src/storage/implementation/MongoWriteBatch.ts

@@ -0,0 +1,172 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+
+/**
+ * Explicitly groups writes with no intervening reads or result dependencies.
+ * Callers retain responsibility for transaction and ordering boundaries.
+ *
+ * On MongoDB 8.0+, this uses the client-level bulkWrite API.
+ *
+ * On MongoDB earlier versions, this uses the the collection-level bulkWrite API, or individual operations.
+ * This is less efficient, but produces the same results.
+ */
+export class MongoWriteBatch {
+  private readonly operations: mongo.AnyClientBulkWriteModel<mongo.Document>[] = [];
+  private readonly deleteChecks: { index: number; check: (count: number) => void }[] = [];
+  private readonly fallback: (() => Promise<unknown>)[] = [];
+
+  private readonly ordered: boolean;
+
+  constructor(
+    private readonly client: mongo.MongoClient,
+    private readonly supportsClientBulkWrite: boolean,
+    private readonly session: mongo.ClientSession | undefined,
+    options: { ordered: boolean }
+  ) {
+    this.ordered = options.ordered;
+  }
+
+  /**
+   * Perform a collection-level bulkWrite.
+   *
+   * On MongoDB 8.0+, these operations join the client-level bulkWrite with the batch's ordering.
+   * The driver may split commands to respect server batch limits.
+   *
+   * On earlier MongoDB versions, this always uses an unordered collection-level bulkWrite.
+   * Operations within this collection bulk must therefore be independent of their order.
+   */
+  bulkWriteUnordered<T extends mongo.Document>(
+    collection: mongo.Collection<T>,
+    operations: mongo.AnyBulkWriteOperation<T>[]
+  ): void {
+    if (operations.length === 0) return;
+    if (!this.supportsClientBulkWrite) {
+      this.fallback.push(() => collection.bulkWrite(operations, { ordered: false, session: this.session }));
+      return;
+    }
+    for (const operation of operations) {
+      // Collection and client models contain the same operation fields, with a
+      // different envelope. Keep the schema checked at the collection boundary.
+      const [name, model] = Object.entries(operation)[0];
+      this.operations.push({ ...model, name, namespace: collection.namespace });
+    }
+  }
+
+  insertOne<T extends mongo.Document>(collection: mongo.Collection<T>, document: mongo.OptionalUnlessRequiredId<T>) {
+    if (!this.supportsClientBulkWrite) {
+      this.fallback.push(() => collection.insertOne(document, { session: this.session }));
+      return;
+    }
+    this.addModel<T>({ name: 'insertOne', namespace: collection.namespace, document: document as mongo.OptionalId<T> });
+  }
+
+  insertMany<T extends mongo.Document>(
+    collection: mongo.Collection<T>,
+    documents: mongo.OptionalUnlessRequiredId<T>[]
+  ) {
+    if (documents.length === 0) return;
+    if (!this.supportsClientBulkWrite) {
+      this.fallback.push(() => collection.insertMany(documents, { session: this.session }));
+      return;
+    }
+    for (const document of documents) {
+      this.addModel<T>({
+        name: 'insertOne',
+        namespace: collection.namespace,
+        document: document as mongo.OptionalId<T>
+      });
+    }
+  }
+
+  deleteMany<T extends mongo.Document>(
+    collection: mongo.Collection<T>,
+    filter: mongo.Filter<T>,
+    checkDeletedCount?: (count: number) => void
+  ) {
+    if (checkDeletedCount) {
+      // Client-bulk results are checked after all writes execute. A failed
+      // invariant must roll back the entire batch, not leave later writes visible.
+      if (!this.session?.inTransaction()) throw new Error('Checked deletes require a transaction');
+    }
+    if (!this.supportsClientBulkWrite) {
+      this.fallback.push(async () => {
+        const result = await collection.deleteMany(filter, { session: this.session });
+        checkDeletedCount?.(result.deletedCount);
+      });
+      return;
+    }
+    if (checkDeletedCount) {
+      this.deleteChecks.push({ index: this.operations.length, check: checkDeletedCount });
+    }
+    this.addModel<T>({ name: 'deleteMany', namespace: collection.namespace, filter });
+  }
+
+  updateOne<T extends mongo.Document>(
+    collection: mongo.Collection<T>,
+    filter: mongo.Filter<T>,
+    update: mongo.UpdateFilter<T> | mongo.Document[],
+    options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
+  ) {
+    if (!this.supportsClientBulkWrite) {
+      this.fallback.push(() => collection.updateOne(filter, update, { ...options, session: this.session }));
+      return;
+    }
+    this.addModel({ name: 'updateOne', namespace: collection.namespace, filter, update, ...options });
+  }
+
+  updateMany<T extends mongo.Document>(
+    collection: mongo.Collection<T>,
+    filter: mongo.Filter<T>,
+    update: mongo.UpdateFilter<T> | mongo.Document[],
+    options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
+  ) {
+    if (!this.supportsClientBulkWrite) {
+      this.fallback.push(() => collection.updateMany(filter, update, { ...options, session: this.session }));
+      return;
+    }
+    this.addModel({ name: 'updateMany', namespace: collection.namespace, filter, update, ...options });
+  }
+
+  private addModel<T extends mongo.Document>(operation: mongo.AnyClientBulkWriteModel<T>) {
+    // The collection-specific methods validate the schema before erasing it here.
+    this.operations.push(operation as unknown as mongo.AnyClientBulkWriteModel<mongo.Document>);
+  }
+
+  /**
+   * Execute the queued writes.
+   *
+   * On MongoDB 8.0+, this uses the client-level bulkWrite API, typically resulting in a single command.
+   * The command may be split if there are a large number of operations.
+   *
+   * On earlier MongoDB versions, this executes each queued operation individually.
+   *
+   * If no operations were queued, this is a no-op.
+   */
+  async execute(): Promise<void> {
+    if (!this.supportsClientBulkWrite) {
+      // Sessions cannot execute concurrent operations within a transaction.
+      for (const write of this.fallback) await write();
+      return;
+    }
+    if (this.operations.length === 0) return;
+
+    try {
+      const result = await this.client.bulkWrite(this.operations, {
+        session: this.session,
+        ordered: this.ordered,
+        verboseResults: this.deleteChecks.length > 0
+      });
+      for (const { index, check } of this.deleteChecks) {
+        const deleted = result.deleteResults?.get(index);
+        if (deleted == null) throw new Error('Missing client bulk delete result');
+        check(deleted.deletedCount);
+      }
+    } catch (error) {
+      // The driver wraps command failures without copying their error labels.
+      // Preserve withTransaction's retry handling for e.g. write conflicts.
+      if (error instanceof mongo.MongoClientBulkWriteError && error.cause instanceof mongo.MongoError) {
+        throw error.cause;
+      }
+      throw error;
+    }
+  }
+}

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -95,7 +95,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
             },
             upsert: true
           }
-        }))
+        })),
+        { ordered: false }
       );
 
       const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
@@ -180,7 +181,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
             upsert: true
           }
         };
-      })
+      }),
+      { ordered: false }
     );
 
     // Fetch the final ids separately so stale requests can still return the

package/src/storage/implementation/common/PersistedBatch.ts

@@ -8,6 +8,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
 import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
 import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
 import { MongoIdSequence } from '../MongoIdSequence.js';
+import { MongoWriteBatch } from '../MongoWriteBatch.js';
 import type { VersionedPowerSyncMongo } from '../db.js';
 import { TaggedBucketParameterDocument } from '../models.js';
 import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
@@ -216,13 +217,13 @@ export abstract class PersistedBatch {
 
   protected abstract get currentDataCount(): number;
 
-  protected abstract flushBucketData(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
 
-  protected abstract flushBucketParameters(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
 
-  protected abstract flushCurrentData(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueCurrentData(writes: MongoWriteBatch): void;
 
-  protected abstract flushBucketStates(session: mongo.ClientSession): Promise<void>;
+  protected abstract queueBucketStates(writes: MongoWriteBatch): void;
 
   protected abstract resetCurrentData(): void;
 
@@ -348,24 +349,27 @@ export abstract class PersistedBatch {
   async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
     const startAt = performance.now();
     let flushedSomething = false;
+    const writes = this.db.createWriteBatch(session, { ordered: false });
     if (this.bucketDataCount > 0) {
       flushedSomething = true;
-      await this.flushBucketData(session);
+      await this.queueBucketData(writes);
     }
     if (this.bucketParameters.length > 0) {
       flushedSomething = true;
-      await this.flushBucketParameters(session);
+      this.queueBucketParameters(writes);
     }
     if (this.currentDataCount > 0) {
       flushedSomething = true;
-      await this.flushCurrentData(session);
+      this.queueCurrentData(writes);
     }
 
     if (this.bucketStates.size > 0) {
       flushedSomething = true;
-      await this.flushBucketStates(session);
+      this.queueBucketStates(writes);
     }
 
+    await writes.execute();
+
     if (flushedSomething) {
       const duration = Math.round(performance.now() - startAt);
       if (options?.oldestUncommittedChange != null) {

package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts

@@ -58,6 +58,10 @@ export abstract class BaseVersionedPowerSyncMongo {
     return this.upstream.notifyCheckpoint();
   }
 
+  createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }) {
+    return this.upstream.createWriteBatch(session, options);
+  }
+
   protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
     return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
   }

package/src/storage/implementation/db.ts

@@ -17,6 +17,7 @@ import {
   SyncRuleDocumentBase,
   WriteCheckpointDocument
 } from './models.js';
+import { MongoWriteBatch } from './MongoWriteBatch.js';
 import {
   BucketDataDocumentV1,
   BucketParameterDocument,
@@ -54,6 +55,19 @@ export class PowerSyncMongo {
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
+  private get clientBulkWriteSupport(): boolean {
+    // The driver exposes topology at runtime, but omits this internal property
+    // from its public types. Reuse its handshake results without another command.
+    const client = this.client as mongo.MongoClient & { topology?: { description: mongo.TopologyDescription } };
+    const servers = [...(client.topology?.description.servers.values() ?? [])];
+    // MongoDB 8.0 introduced client bulkWrite at wire version 25. Unknown / mixed-version
+    // topologies use collection writes until all known servers support it.
+    return servers.length > 0 && servers.every((server) => server.maxWireVersion >= 25);
+  }
+
+  createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }): MongoWriteBatch {
+    return new MongoWriteBatch(this.client, this.clientBulkWriteSupport, session, options);
+  }
 
   constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions) {
     this.client = client;
@@ -67,6 +81,7 @@ export class PowerSyncMongo {
     this.bucket_data = db.collection('bucket_data');
     this.bucket_parameters = db.collection('bucket_parameters');
     this.op_id_sequence = db.collection('op_id_sequence');
+    // Uses legacy terminology instead of "sync config" for backwards-compatibility
     this.sync_rules = db.collection('sync_rules');
     this.source_tables = db.collection('source_tables');
     this.custom_write_checkpoints = db.collection('custom_write_checkpoints');

package/src/storage/implementation/models.ts

@@ -308,6 +308,7 @@ export interface CustomWriteCheckpointDocument {
   _id: bson.ObjectId;
   user_id: string;
   checkpoint: bigint;
+  // Uses the legacy field name "sync_rules_id" for backwards-compatibility. See AGENTS.md.
   sync_rules_id: number;
   /**
    * Unlike managed write checkpoints, custom write checkpoints are flushed together with