@powersync/service-module-mongodb-storage 0.23.1 → 0.24.0

package/src/storage/implementation/MongoOpIdAllocator.ts

@@ -2,6 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { VersionedPowerSyncMongo } from './db.js';
 import { MongoIdSequence, OpIdRange } from './MongoIdSequence.js';
+import { REPLICATION_WRITE_OPTIONS } from './MongoReplicationOptions.js';
 
 const RESERVATION_SIZE = 65_536n;
 const LOW_WATER_MARK = 16_384n;
@@ -23,13 +24,19 @@ export class MongoOpIdAllocator {
   }
 
   sequence(persistedOp: bigint): MongoIdSequence {
-    // The stream is fenced before reading this head. Another writer may have
-    // committed since this allocator was last used, including in another process.
-    // Never reuse reserved IDs below that durable head.
+    // Transactional writers pass the fenced stream head. Prepared v3/v4 writers
+    // pass their last ID within this publication (initially zero). Their per-lease
+    // ranges are globally disjoint, and assigned IDs are consumed before publish.
+    // Publication asserts that its final ID advances the fenced persisted head.
     return new MongoIdSequence(persistedOp, this.ranges.slice());
   }
 
-  committed(lastOp: bigint) {
+  /**
+   * Remove IDs assigned to sealed work, even if publication is later abandoned.
+   * This only consumes allocation capacity; it does not advance durable progress.
+   * Keep the unused tail available to the next publication under this lease.
+   */
+  consume(lastOp: bigint) {
     this.ranges = this.ranges
       .filter((range) => range.end > lastOp)
       .map((range) => ({ start: range.start > lastOp ? range.start : lastOp + 1n, end: range.end }));
@@ -86,7 +93,7 @@ export class MongoOpIdAllocator {
             }
           }
         ],
-        { upsert: true, returnDocument: 'before', writeConcern: { w: 'majority' } }
+        { upsert: true, returnDocument: 'before', ...REPLICATION_WRITE_OPTIONS }
       );
       const last = previous?.op_id ?? 0n;
       if (last > MAX_OP_ID - RESERVATION_SIZE) {

package/src/storage/implementation/MongoParsedSyncConfigSet.ts

@@ -1,6 +1,10 @@
 import * as sqlite from 'node:sqlite';
 
-import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  ReplicationAbortedError,
+  ReplicationAssertionError,
+  ServiceAssertionError
+} from '@powersync/lib-services-framework';
 import {
   BucketDefinitionMapping,
   MultiSyncConfigBucketDefinitionMapping,
@@ -26,6 +30,41 @@ import { StorageConfig } from './models.js';
 import { MongoHydrationState } from './MongoHydrationState.js';
 
 export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet {
+  /** Config definitions/mappings are immutable; only their embedded lifecycle state changes. */
+  assertPublicationConfigs(states: readonly { _id: { toHexString(): string }; state: storage.SyncRuleState }[]): void {
+    const configured = new Set(this.#configsWithMapping.map((config) => config.syncConfigId));
+    const live = new Set(
+      states
+        .filter((state) =>
+          [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ERRORED].includes(
+            state.state
+          )
+        )
+        .map((state) => state._id.toHexString())
+    );
+    // Snapshot/metadata writers may intentionally select only one processing
+    // config while another active config serves clients. Full-stream checkpoint
+    // writers separately require every replicating config in their checkpoint guard.
+    if (![...live].some((id) => configured.has(id))) {
+      throw new ReplicationAbortedError('Replication writer must reload the selected sync configs');
+    }
+    const mappings = this.#configsWithMapping
+      .filter((config) => live.has(config.syncConfigId!))
+      .map((config) => config.mapping!);
+    // A stopped selected config is harmless only when all its persisted outputs
+    // are still owned by live selected configs. Cleanup may remove every other id.
+    const activeBuckets = new Set(mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()));
+    const activeParameters = new Set(mappings.flatMap((mapping) => mapping.allParameterIndexIds()));
+    const activeEvents = new Set(mappings.flatMap((mapping) => mapping.allEventDefinitionIds()));
+    if (
+      this.mapping.allBucketDefinitionIds().some((id) => !activeBuckets.has(id)) ||
+      this.mapping.allParameterIndexIds().some((id) => !activeParameters.has(id)) ||
+      this.mapping.allEventDefinitionIds().some((id) => !activeEvents.has(id))
+    ) {
+      throw new ReplicationAbortedError('Replication writer mappings are no longer live; reload the sync configs');
+    }
+  }
+
   public readonly hydrationState: HydrationState;
   public readonly syncConfigs: SyncConfigWithErrors[];
   public readonly replicationStreamName: string;

package/src/storage/implementation/MongoReplicationOptions.ts

@@ -0,0 +1,28 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+
+/**
+ * Read options for replication preparation outside publication transactions.
+ * Preparation produces writes that publication replays without rereading their
+ * source records, so it must not depend on data that can later be rolled back.
+ * Majority reads, majority writes and the lease's shared causal session provide
+ * durable causal ordering. A causal session alone does not select majority read
+ * concern; the ordinary default is local. Primary is the usual routing default,
+ * made explicit here so preparation does not inherit a different preference.
+ * Set transaction read concern at transaction level, not through these options.
+ */
+export const REPLICATION_READ_OPTIONS = {
+  readConcern: 'majority',
+  readPreference: 'primary'
+} as const satisfies Pick<mongo.CommandOperationOptions, 'readConcern' | 'readPreference'>;
+
+/**
+ * Acknowledge durable replication writes before using allocated IDs, reporting
+ * publication success or preparing subsequent work in the shared causal session.
+ * Apply at transaction level for publication, and at operation level for writes
+ * outside transactions (ID reservations, metadata and post-commit cleanup).
+ * Resume-position hints deliberately use w: 1 instead: losing a hint causes
+ * source replay, while row publication and checkpoints still require durability.
+ */
+export const REPLICATION_WRITE_OPTIONS = {
+  writeConcern: { w: 'majority' }
+} as const satisfies Pick<mongo.TransactionOptions, 'writeConcern'>;

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -34,7 +34,6 @@ import {
   MongoGetCheckpointChangesOptions,
   MongoSyncBucketStorageCheckpoint
 } from './common/MongoSyncBucketStorageCheckpoint.js';
-import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js';
 import type { VersionedPowerSyncMongo } from './db.js';
 import { StorageConfig } from './models.js';
 import { MongoBucketBatchOptions } from './MongoBucketBatch.js';
@@ -46,6 +45,7 @@ import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStre
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
 import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
+import { DEFAULT_INLINE_THRESHOLD_BYTES } from './v3/PersistedBatchV3.js';
 
 export interface MongoSyncBucketStorageOptions {
   checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
@@ -278,7 +278,6 @@ export abstract class MongoSyncBucketStorage
     const replicationLock = this.requireReplicationLock();
     return {
       logger: options.logger ?? this.logger,
-      db: this.db,
       parsedSyncConfig: this.getParsedSyncConfigSet(options),
       replicationStreamId: this.replicationStreamId,
       replicationStreamName: this.replicationStreamName,
@@ -464,7 +463,7 @@ export abstract class MongoSyncBucketStorage
     }
 
     const stream = mergeAsyncIterables(
-      [this.checkpointChangesStream(signal), this.checkpointTimeoutStream(signal)],
+      [(signal) => this.checkpointChangesStream(signal), (signal) => this.checkpointTimeoutStream(signal)],
       signal
     );
 

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -12,8 +12,15 @@ const LOCK_DURATION_MS = 60 * 1000;
  * processes that replication stream at a time.
  */
 export class MongoSyncRulesLock implements storage.ReplicationLock {
-  /** FIFO admission shared by this lease's writers; database fencing still enforces ownership. */
+  /** Public writer operations hold this mutex through preparation, publication and cleanup. */
   readonly writerMutex = new utils.Mutex();
+  /**
+   * All writers use this causal session sequentially under writerMutex. Reusing
+   * it carries earlier writes into later majority reads without copying clocks.
+   * Only lease teardown ends it; disposing one writer must not affect the rest.
+   * Lease acquisition and renewal use independent operations and never wait here.
+   */
+  readonly session: mongo.ClientSession;
 
   private readonly abort = new AbortController();
   readonly signal = this.abort.signal;
@@ -71,37 +78,42 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   }
 
   constructor(
-    private db: VersionedPowerSyncMongo,
+    public readonly db: VersionedPowerSyncMongo,
     public sync_rules_id: number,
     public readonly lock_id: string,
     private logger: Logger
   ) {
+    this.session = db.client.startSession({ causalConsistency: true });
     this.refreshInterval = setInterval(async () => {
       try {
         await this.refresh();
       } catch (e) {
-        this.abort.abort(e);
+        this.invalidate(e);
         this.logger.error('Failed to refresh lock', e);
-        clearInterval(this.refreshInterval);
       }
     }, 30_130);
   }
 
   async release(): Promise<void> {
-    this.abort.abort(new Error('Replication lock released'));
-    clearInterval(this.refreshInterval);
-    const result = await this.db.sync_rules.updateOne(
-      {
-        _id: this.sync_rules_id,
-        'lock.id': this.lock_id
-      },
-      {
-        $unset: { lock: 1 }
+    this.invalidate(new Error('Replication lock released'));
+    try {
+      const result = await this.db.sync_rules.updateOne(
+        {
+          _id: this.sync_rules_id,
+          'lock.id': this.lock_id
+        },
+        {
+          $unset: { lock: 1 }
+        }
+      );
+      if (result.modifiedCount == 0) {
+        // Log and ignore
+        this.logger.warn(`Lock already released: ${this.sync_rules_id}/${this.lock_id}`);
       }
-    );
-    if (result.modifiedCount == 0) {
-      // Log and ignore
-      this.logger.warn(`Lock already released: ${this.sync_rules_id}/${this.lock_id}`);
+    } finally {
+      // Let an active operation finish/abort before ending its session. Queued
+      // writers check the aborted lease after acquiring the mutex and do no I/O.
+      await this.writerMutex.exclusiveLock(() => this.session.endSession());
     }
   }
 
@@ -109,6 +121,16 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
     this.signal.throwIfAborted();
   }
 
+  /**
+   * Stop all local writers, including queued operations, when continuing this
+   * replication attempt would be unsafe. The job still releases the database
+   * lease during teardown; otherwise it expires after renewal stops.
+   */
+  invalidate(reason: unknown): void {
+    this.abort.abort(reason);
+    clearInterval(this.refreshInterval);
+  }
+
   static ownerFilter(streamId: number, lock: MongoSyncRulesLock) {
     lock.throwIfAborted();
     return { _id: streamId, 'lock.id': lock.lock_id };

package/src/storage/implementation/common/BatchOperationBuffer.ts

@@ -0,0 +1,353 @@
+import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework';
+import { InternalOpId, storage, utils } from '@powersync/service-core';
+import { JSONBig } from '@powersync/service-jsonbig';
+import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
+import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
+import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
+import { TaggedBucketParameterDocument } from '../models.js';
+import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
+import { BucketStateUpdate, SaveBucketDataOptions } from './PersistedBatch.js';
+import { PublicationStats } from './PreparedPublication.js';
+import { SourceRecordBucketState } from './SourceRecordStore.js';
+
+/**
+ * Maximum size of operations we write in a single transaction.
+ *
+ * It's tricky to find the exact limit, but from experience, over 100MB
+ * can cause an error:
+ * > transaction is too large and will not fit in the storage engine cache
+ *
+ * Additionally, unbounded size here can balloon our memory usage in some edge
+ * cases.
+ *
+ * When we reach this threshold, we commit the transaction and start a new one.
+ */
+const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
+
+/**
+ * Limit number of documents to write in a single transaction.
+ *
+ * This has an effect on error message size in some cases.
+ */
+const MAX_TRANSACTION_DOC_COUNT = 2_000;
+
+/**
+ * Shared operation generation, size accounting and diagnostics. Each storage
+ * version owns this buffer and decides when to write, retain or clear its data.
+ * No transaction, sealing or payload-preparation state lives here.
+ */
+export class BatchOperationBuffer {
+  bucketData: BucketDataDoc[] = [];
+  bucketParameters: TaggedBucketParameterDocument[] = [];
+  bucketStates = new Map<string, BucketStateUpdate>();
+
+  /**
+   * For debug logging only.
+   */
+  debugLastOpId: InternalOpId | null = null;
+
+  /**
+   * Very rough estimate of transaction size.
+   */
+  currentSize: number;
+
+  constructor(
+    private readonly group_id: number,
+    writtenSize: number,
+    private readonly definitions: {
+      checkDefinitionId(id: BucketDefinitionId | null): BucketDefinitionId;
+      getBucketDefinitionId(source: BucketDataSource): BucketDefinitionId;
+    },
+    private readonly logger: Logger = defaultLogger
+  ) {
+    this.currentSize = writtenSize;
+  }
+
+  saveBucketData(options: SaveBucketDataOptions) {
+    const remaining_buckets = new Map<string, SourceRecordBucketState>();
+    for (let bucket of options.before_buckets) {
+      const mapped: SourceRecordBucketState = {
+        bucket: bucket.bucket,
+        definitionId: this.definitions.checkDefinitionId(bucket.definitionId),
+        id: bucket.id,
+        table: bucket.table
+      };
+      if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId!)) {
+        // The bucket definition is not active anymore.
+        // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
+        continue;
+      }
+      remaining_buckets.set(currentBucketKey(mapped), mapped);
+    }
+
+    const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
+    const dchecksum = BigInt(utils.hashDelete(subkey));
+
+    for (const evaluated of options.evaluated) {
+      const definitionId = this.definitions.getBucketDefinitionId(evaluated.source);
+      const key = currentBucketKey({
+        definitionId: definitionId,
+        bucket: evaluated.bucket,
+        table: evaluated.table,
+        id: evaluated.id
+      });
+
+      const recordData = JSONBig.stringify(evaluated.data);
+      const checksum = utils.hashData(evaluated.table, evaluated.id, recordData);
+      if (recordData.length > MAX_ROW_SIZE) {
+        const message = `Row ${key} too large: ${recordData.length} bytes. Removing.`;
+        this.logger.error(message);
+        continue;
+      }
+
+      remaining_buckets.delete(key);
+      const byteEstimate = recordData.length + 200;
+      this.currentSize += byteEstimate;
+
+      const op_id = options.op_seq.next();
+      this.debugLastOpId = op_id;
+
+      this.addBucketDataPut({
+        bucketKey: {
+          bucket: evaluated.bucket,
+          definitionId: definitionId,
+          replicationStreamId: this.group_id
+        },
+        op_id,
+        bucket: evaluated.bucket,
+        sourceTableId: options.table.id,
+        sourceKey: options.sourceKey,
+        subkey,
+        table: evaluated.table,
+        rowId: evaluated.id,
+        checksum: BigInt(checksum),
+        data: recordData
+      });
+      this.incrementBucket(definitionId, evaluated.bucket, op_id, byteEstimate);
+    }
+
+    for (let bucket of remaining_buckets.values()) {
+      const definitionId = bucket.definitionId!;
+      const op_id = options.op_seq.next();
+      this.debugLastOpId = op_id;
+
+      this.addBucketDataRemove({
+        bucketKey: {
+          replicationStreamId: this.group_id,
+          definitionId,
+          bucket: bucket.bucket
+        },
+        op_id,
+        sourceTableId: options.table.id,
+        sourceKey: options.sourceKey,
+        subkey,
+        table: bucket.table,
+        rowId: bucket.id,
+        checksum: dchecksum
+      });
+      this.currentSize += 200;
+      this.incrementBucket(definitionId, bucket.bucket, op_id, 200);
+    }
+  }
+
+  /**
+   * New bucket operations persist protocol subkeys during replication. Older
+   * V1 documents may not have one, so sync reads retain a fallback.
+   */
+  persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string {
+    return replicaIdToSubkey(table, key);
+  }
+
+  get bucketDataCount(): number {
+    return this.bucketData.length;
+  }
+
+  incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number) {
+    const key = `${definitionId ?? ''}:${bucket}`;
+    let existingState = this.bucketStates.get(key);
+    if (existingState) {
+      existingState.lastOp = op_id;
+      existingState.incrementCount += 1;
+      existingState.incrementBytes += bytes;
+    } else {
+      this.bucketStates.set(key, {
+        definitionId,
+        bucket,
+        lastOp: op_id,
+        incrementCount: 1,
+        incrementBytes: bytes,
+        incrementChunks: 0
+      });
+    }
+  }
+
+  /**
+   * V3 persists operations in chunks. Keep this separate from incrementBucket:
+   * operation counts are known while evaluating rows, while chunk counts and
+   * exact persisted bytes are only known after the writer has chunked a flush.
+   */
+  incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) {
+    const key = `${definitionId ?? ''}:${bucket}`;
+    const existingState = this.bucketStates.get(key);
+    if (existingState != null) {
+      existingState.incrementChunks += 1;
+      existingState.incrementBytes += bytes;
+    }
+  }
+
+  /**
+   * V3's compactor calculates byte deltas from persisted chunk metadata.
+   * Replace the writer's per-operation estimate with those exact sizes before
+   * flushing the corresponding bucket state.
+   */
+  resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) {
+    const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
+    if (state != null) {
+      state.incrementBytes = 0;
+    }
+  }
+
+  addBucketDataPut(options: {
+    op_id: InternalOpId;
+    bucketKey: BucketKey;
+    bucket: string;
+    sourceTableId: storage.SourceTable['id'];
+    sourceKey: storage.ReplicaId;
+    subkey: string;
+    table: string;
+    rowId: string;
+    checksum: bigint;
+    data: string;
+  }) {
+    this.bucketData.push({
+      bucketKey: options.bucketKey,
+      o: options.op_id,
+      op: 'PUT',
+      source_table: mongoTableId(options.sourceTableId),
+      source_key: options.sourceKey,
+      subkey: options.subkey,
+      table: options.table,
+      row_id: options.rowId,
+      checksum: options.checksum,
+      data: options.data
+    });
+  }
+
+  addBucketDataRemove(options: {
+    op_id: InternalOpId;
+    bucketKey: BucketKey;
+    sourceTableId: storage.SourceTable['id'];
+    sourceKey: storage.ReplicaId;
+    subkey: string;
+    table: string;
+    rowId: string;
+    checksum: bigint;
+  }) {
+    this.bucketData.push({
+      bucketKey: options.bucketKey,
+      o: options.op_id,
+      op: 'REMOVE',
+      source_table: mongoTableId(options.sourceTableId),
+      source_key: options.sourceKey,
+      subkey: options.subkey,
+      table: options.table,
+      row_id: options.rowId,
+      checksum: options.checksum,
+      data: null
+    });
+  }
+
+  shouldFlushTransaction(currentDataCount: number) {
+    return (
+      this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
+      this.bucketDataCount >= MAX_TRANSACTION_DOC_COUNT ||
+      currentDataCount >= MAX_TRANSACTION_DOC_COUNT ||
+      this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT
+    );
+  }
+
+  /** Adopt one fully evaluated row; unfinished rows never mutate this buffer. */
+  append(row: BatchOperationBuffer): void {
+    // A single high-fanout row may exceed a normal batch target. Avoid the
+    // JavaScript argument-count limit while retaining its operation order.
+    for (const operation of row.bucketData) {
+      this.bucketData.push(operation);
+    }
+    for (const parameter of row.bucketParameters) {
+      this.bucketParameters.push(parameter);
+    }
+    for (const [key, state] of row.bucketStates) {
+      const previous = this.bucketStates.get(key);
+      this.bucketStates.set(key, {
+        ...state,
+        incrementCount: (previous?.incrementCount ?? 0) + state.incrementCount,
+        incrementBytes: (previous?.incrementBytes ?? 0) + state.incrementBytes,
+        incrementChunks: (previous?.incrementChunks ?? 0) + state.incrementChunks
+      });
+    }
+    this.currentSize += row.currentSize;
+    this.debugLastOpId = row.debugLastOpId ?? this.debugLastOpId;
+  }
+
+  clear(): void {
+    this.bucketData = [];
+    this.bucketParameters = [];
+    this.bucketStates.clear();
+    this.currentSize = 0;
+    this.debugLastOpId = null;
+  }
+
+  logFlush(startAt: number, currentDataCount: number, options?: storage.BucketBatchCommitOptions): PublicationStats {
+    const flushedSomething =
+      this.bucketDataCount > 0 ||
+      this.bucketParameters.length > 0 ||
+      currentDataCount > 0 ||
+      this.bucketStates.size > 0;
+    if (flushedSomething) {
+      const duration = Math.round(performance.now() - startAt);
+      if (options?.oldestUncommittedChange != null) {
+        const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
+
+        this.logger.info(
+          `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
+            currentDataCount
+          } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`,
+          {
+            flushed: {
+              duration: duration,
+              size: this.currentSize,
+              bucket_ops_count: this.bucketDataCount,
+              parameter_indexes_count: this.bucketParameters.length,
+              source_records_count: currentDataCount,
+              replication_lag_seconds: replicationLag
+            }
+          }
+        );
+      } else {
+        this.logger.info(
+          `Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${
+            currentDataCount
+          } records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`,
+          {
+            flushed: {
+              duration: duration,
+              size: this.currentSize,
+              bucket_ops_count: this.bucketDataCount,
+              parameter_indexes_count: this.bucketParameters.length,
+              source_records_count: currentDataCount
+            }
+          }
+        );
+      }
+    }
+
+    const stats = {
+      bucketDataCount: this.bucketDataCount,
+      parameterDataCount: this.bucketParameters.length,
+      currentDataCount: currentDataCount,
+      flushedAny: flushedSomething
+    };
+
+    return stats;
+  }
+}