@powersync/service-module-mongodb-storage 0.13.2 → 0.15.0

package/src/storage/implementation/MongoChecksums.ts

@@ -2,6 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import {
   addPartialChecksums,
   bson,
+  BucketChecksumRequest,
   BucketChecksum,
   ChecksumCache,
   ChecksumMap,
@@ -12,7 +13,8 @@ import {
   PartialChecksumMap,
   PartialOrFullChecksum
 } from '@powersync/service-core';
-import { PowerSyncMongo } from './db.js';
+import { VersionedPowerSyncMongo } from './db.js';
+import { StorageConfig } from './models.js';
 
 /**
  * Checksum calculation options, primarily for tests.
@@ -27,6 +29,8 @@ export interface MongoChecksumOptions {
    * Limit on the number of documents to calculate a checksum on at a time.
    */
   operationBatchLimit?: number;
+
+  storageConfig: StorageConfig;
 }
 
 const DEFAULT_BUCKET_BATCH_LIMIT = 200;
@@ -43,12 +47,15 @@ const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
  */
 export class MongoChecksums {
   private _cache: ChecksumCache | undefined;
+  private readonly storageConfig: StorageConfig;
 
   constructor(
-    private db: PowerSyncMongo,
+    private db: VersionedPowerSyncMongo,
     private group_id: number,
-    private options?: MongoChecksumOptions
-  ) {}
+    private options: MongoChecksumOptions
+  ) {
+    this.storageConfig = options.storageConfig;
+  }
 
   /**
    * Lazy-instantiated cache.
@@ -68,7 +75,7 @@ export class MongoChecksums {
    * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
    * the database (bucket_state + bucket_data).
    */
-  async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap> {
+  async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
     return this.cache.getChecksumMap(checkpoint, buckets);
   }
 
@@ -222,6 +229,11 @@ export class MongoChecksums {
         });
       }
 
+      // Historically, checksum may be stored as 'int' or 'double'.
+      // More recently, this should be a 'long'.
+      // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
+      const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' };
+
       // Aggregate over a max of `batchLimit` operations at a time.
       // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
       // Then we'll do three batches:
@@ -245,10 +257,7 @@ export class MongoChecksums {
             {
               $group: {
                 _id: '$_id.b',
-                // Historically, checksum may be stored as 'int' or 'double'.
-                // More recently, this should be a 'long'.
-                // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-                checksum_total: { $sum: { $toLong: '$checksum' } },
+                checksum_total: { $sum: checksumLong },
                 count: { $sum: 1 },
                 has_clear_op: {
                   $max: {
@@ -290,6 +299,7 @@ export class MongoChecksums {
           const req = requests.get(bucket);
           requests.set(bucket, {
             bucket,
+            source: req!.source,
             start: doc.last_op,
             end: req!.end
           });

package/src/storage/implementation/MongoCompactor.ts

@@ -9,7 +9,7 @@ import {
   utils
 } from '@powersync/service-core';
 
-import { PowerSyncMongo } from './db.js';
+import { VersionedPowerSyncMongo } from './db.js';
 import { BucketDataDocument, BucketDataKey, BucketStateDocument } from './models.js';
 import { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js';
 import { cacheKey } from './OperationBatch.js';
@@ -63,6 +63,7 @@ const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 const DEFAULT_MIN_BUCKET_CHANGES = 10;
 const DEFAULT_MIN_CHANGE_RATIO = 0.1;
+const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
 
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
@@ -84,19 +85,19 @@ export class MongoCompactor {
 
   constructor(
     private storage: MongoSyncBucketStorage,
-    private db: PowerSyncMongo,
-    options?: MongoCompactOptions
+    private db: VersionedPowerSyncMongo,
+    options: MongoCompactOptions
   ) {
     this.group_id = storage.group_id;
-    this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
-    this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
-    this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
-    this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
-    this.minBucketChanges = options?.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
-    this.minChangeRatio = options?.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
-    this.maxOpId = options?.maxOpId ?? 0n;
-    this.buckets = options?.compactBuckets;
-    this.signal = options?.signal;
+    this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
+    this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
+    this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
+    this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
+    this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
+    this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
+    this.maxOpId = options.maxOpId ?? 0n;
+    this.buckets = options.compactBuckets;
+    this.signal = options.signal;
   }
 
   /**
@@ -538,31 +539,60 @@ export class MongoCompactor {
     let lastId = { g: this.group_id, b: new mongo.MinKey() as any };
     const maxId = { g: this.group_id, b: new mongo.MaxKey() as any };
     while (true) {
-      const batch = await this.db.bucket_state
-        .find(
-          {
-            _id: { $gt: lastId, $lt: maxId },
-            'estimate_since_compact.count': { $gte: options.minBucketChanges }
-          },
-          {
-            projection: {
-              _id: 1,
-              estimate_since_compact: 1,
-              compacted_state: 1
+      // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, we use an aggregation pipeline
+      // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria, rather than limiting
+      // on the output number.
+      const [result] = await this.db.bucket_state
+        .aggregate<{
+          buckets: Pick<BucketStateDocument, '_id' | 'estimate_since_compact' | 'compacted_state'>[];
+          cursor: Pick<BucketStateDocument, '_id'>[];
+        }>(
+          [
+            {
+              $match: {
+                _id: { $gt: lastId, $lt: maxId }
+              }
             },
-            sort: {
-              _id: 1
+            {
+              $sort: { _id: 1 }
             },
-            limit: 2000,
-            maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
-          }
+            {
+              // Scan a fixed number of docs each query so sparse matches don't block progress.
+              $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE
+            },
+            {
+              $facet: {
+                // This is the results for the batch
+                buckets: [
+                  {
+                    $match: {
+                      'estimate_since_compact.count': { $gte: options.minBucketChanges }
+                    }
+                  },
+                  {
+                    $project: {
+                      _id: 1,
+                      estimate_since_compact: 1,
+                      compacted_state: 1
+                    }
+                  }
+                ],
+                // This is used for the next query.
+                cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
+              }
+            }
+          ],
+          { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS }
         )
         .toArray();
-      if (batch.length == 0) {
+
+      const cursor = result?.cursor?.[0];
+      if (cursor == null) {
         break;
       }
-      lastId = batch[batch.length - 1]._id;
-      const mapped = batch.map((b) => {
+      lastId = cursor._id;
+
+      const mapped = (result?.buckets ?? []).map((b) => {
         const updatedCount = b.estimate_since_compact?.count ?? 0;
         const totalCount = (b.compacted_state?.count ?? 0) + updatedCount;
         const updatedBytes = b.estimate_since_compact?.bytes ?? 0;
@@ -632,6 +662,7 @@ export class MongoCompactor {
       buckets.map((bucket) => {
         return {
           bucket,
+          source: {} as any,
           end: this.maxOpId
         };
       })

package/src/storage/implementation/MongoParameterCompactor.ts

@@ -1,8 +1,8 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { logger } from '@powersync/lib-services-framework';
 import { bson, CompactOptions, InternalOpId } from '@powersync/service-core';
 import { LRUCache } from 'lru-cache';
-import { PowerSyncMongo } from './db.js';
-import { mongo } from '@powersync/lib-service-mongodb';
+import { VersionedPowerSyncMongo } from './db.js';
 import { BucketParameterDocument } from './models.js';
 
 /**
@@ -14,7 +14,7 @@ import { BucketParameterDocument } from './models.js';
  */
 export class MongoParameterCompactor {
   constructor(
-    private db: PowerSyncMongo,
+    private db: VersionedPowerSyncMongo,
     private group_id: number,
     private checkpoint: InternalOpId,
     private options: CompactOptions

package/src/storage/implementation/MongoPersistedSyncRulesContent.ts

@@ -1,52 +1,38 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js';
 import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
-import { PowerSyncMongo } from './db.js';
-import { SyncRuleDocument } from './models.js';
-
-export class MongoPersistedSyncRulesContent implements storage.PersistedSyncRulesContent {
-  public readonly slot_name: string;
-
-  public readonly id: number;
-  public readonly sync_rules_content: string;
-  public readonly last_checkpoint_lsn: string | null;
-  public readonly last_fatal_error: string | null;
-  public readonly last_fatal_error_ts: Date | null;
-  public readonly last_keepalive_ts: Date | null;
-  public readonly last_checkpoint_ts: Date | null;
-  public readonly active: boolean;
+import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js';
+import { getMongoStorageConfig, SyncRuleDocument } from './models.js';
 
+export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent {
   public current_lock: MongoSyncRulesLock | null = null;
 
   constructor(
     private db: PowerSyncMongo,
     doc: mongo.WithId<SyncRuleDocument>
   ) {
-    this.id = doc._id;
-    this.sync_rules_content = doc.content;
-    this.last_checkpoint_lsn = doc.last_checkpoint_lsn;
-    // Handle legacy values
-    this.slot_name = doc.slot_name ?? `powersync_${this.id}`;
-    this.last_fatal_error = doc.last_fatal_error;
-    this.last_fatal_error_ts = doc.last_fatal_error_ts;
-    this.last_checkpoint_ts = doc.last_checkpoint_ts;
-    this.last_keepalive_ts = doc.last_keepalive_ts;
-    this.active = doc.state == 'ACTIVE';
+    super({
+      id: doc._id,
+      sync_rules_content: doc.content,
+      compiled_plan: doc.serialized_plan ?? null,
+      last_checkpoint_lsn: doc.last_checkpoint_lsn,
+      // Handle legacy values
+      slot_name: doc.slot_name ?? `powersync_${doc._id}`,
+      last_fatal_error: doc.last_fatal_error,
+      last_fatal_error_ts: doc.last_fatal_error_ts,
+      last_checkpoint_ts: doc.last_checkpoint_ts,
+      last_keepalive_ts: doc.last_keepalive_ts,
+      active: doc.state == 'ACTIVE',
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION
+    });
   }
 
-  parsed(options: storage.ParseSyncRulesOptions) {
-    return new MongoPersistedSyncRules(
-      this.id,
-      SqlSyncRules.fromYaml(this.sync_rules_content, options),
-      this.last_checkpoint_lsn,
-      this.slot_name
-    );
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
   }
 
   async lock() {
-    const lock = await MongoSyncRulesLock.createLock(this.db, this);
+    const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this);
     this.current_lock = lock;
     return lock;
   }

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -31,16 +31,25 @@ import { LRUCache } from 'lru-cache';
 import * as timers from 'timers/promises';
 import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js';
+import { VersionedPowerSyncMongo } from './db.js';
+import {
+  BucketDataDocument,
+  BucketDataKey,
+  BucketStateDocument,
+  SourceKey,
+  SourceTableDocument,
+  StorageConfig
+} from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoParameterCompactor } from './MongoParameterCompactor.js';
+import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 
 export interface MongoSyncBucketStorageOptions {
-  checksumOptions?: MongoChecksumOptions;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  storageConfig: StorageConfig;
 }
 
 /**
@@ -58,7 +67,7 @@ export class MongoSyncBucketStorage
   extends BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
 {
-  private readonly db: PowerSyncMongo;
+  private readonly db: VersionedPowerSyncMongo;
   readonly checksums: MongoChecksums;
 
   private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined;
@@ -67,14 +76,17 @@ export class MongoSyncBucketStorage
   constructor(
     public readonly factory: MongoBucketStorage,
     public readonly group_id: number,
-    private readonly sync_rules: storage.PersistedSyncRulesContent,
+    private readonly sync_rules: MongoPersistedSyncRulesContent,
     public readonly slot_name: string,
-    writeCheckpointMode?: storage.WriteCheckpointMode,
-    options?: MongoSyncBucketStorageOptions
+    writeCheckpointMode: storage.WriteCheckpointMode | undefined,
+    options: MongoSyncBucketStorageOptions
   ) {
     super();
-    this.db = factory.db;
-    this.checksums = new MongoChecksums(this.db, this.group_id, options?.checksumOptions);
+    this.db = factory.db.versioned(sync_rules.getStorageConfig());
+    this.checksums = new MongoChecksums(this.db, this.group_id, {
+      ...options.checksumOptions,
+      storageConfig: options?.storageConfig
+    });
     this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
       db: this.db,
       mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
@@ -175,7 +187,6 @@ export class MongoSyncBucketStorage
       slotName: this.slot_name,
       lastCheckpointLsn: checkpoint_lsn,
       resumeFromLsn: maxLsn(checkpoint_lsn, doc?.snapshot_lsn),
-      noCheckpointBeforeLsn: doc?.no_checkpoint_before ?? options.zeroLSN,
       keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
       storeCurrentData: options.storeCurrentData,
       skipExistingRows: options.skipExistingRows ?? false,
@@ -361,19 +372,20 @@ export class MongoSyncBucketStorage
 
   async *getBucketDataBatch(
     checkpoint: utils.InternalOpId,
-    dataBuckets: Map<string, InternalOpId>,
+    dataBuckets: storage.BucketDataRequest[],
     options?: storage.BucketDataBatchOptions
   ): AsyncIterable<storage.SyncBucketDataChunk> {
-    if (dataBuckets.size == 0) {
+    if (dataBuckets.length == 0) {
       return;
     }
     let filters: mongo.Filter<BucketDataDocument>[] = [];
+    const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
 
     if (checkpoint == null) {
       throw new ServiceAssertionError('checkpoint is null');
     }
     const end = checkpoint;
-    for (let [name, start] of dataBuckets.entries()) {
+    for (let { bucket: name, start } of dataBuckets) {
       filters.push({
         _id: {
           $gt: {
@@ -466,7 +478,7 @@ export class MongoSyncBucketStorage
         }
 
         if (start == null) {
-          const startOpId = dataBuckets.get(bucket);
+          const startOpId = bucketMap.get(bucket);
           if (startOpId == null) {
             throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
           }
@@ -508,7 +520,10 @@ export class MongoSyncBucketStorage
     }
   }
 
-  async getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap> {
+  async getChecksums(
+    checkpoint: utils.InternalOpId,
+    buckets: storage.BucketChecksumRequest[]
+  ): Promise<utils.ChecksumMap> {
     return this.checksums.getChecksums(checkpoint, buckets);
   }
 
@@ -565,7 +580,7 @@ export class MongoSyncBucketStorage
   async clear(options?: storage.ClearStorageOptions): Promise<void> {
     while (true) {
       if (options?.signal?.aborted) {
-        throw new ReplicationAbortedError('Aborted clearing data');
+        throw new ReplicationAbortedError('Aborted clearing data', options.signal.reason);
       }
       try {
         await this.clearIteration();
@@ -620,7 +635,7 @@ export class MongoSyncBucketStorage
       { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
     );
 
-    await this.db.current_data.deleteMany(
+    await this.db.common_current_data.deleteMany(
       {
         _id: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
       },

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -2,7 +2,7 @@ import crypto from 'crypto';
 
 import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
-import { PowerSyncMongo } from './db.js';
+import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js';
 
 /**
  * Manages a lock on a sync rules document, so that only one process
@@ -12,7 +12,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   private readonly refreshInterval: NodeJS.Timeout;
 
   static async createLock(
-    db: PowerSyncMongo,
+    db: VersionedPowerSyncMongo,
     sync_rules: storage.PersistedSyncRulesContent
   ): Promise<MongoSyncRulesLock> {
     const lockId = crypto.randomBytes(8).toString('hex');
@@ -52,7 +52,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
   }
 
   constructor(
-    private db: PowerSyncMongo,
+    private db: VersionedPowerSyncMongo,
     public sync_rules_id: number,
     private lock_id: string
   ) {

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -1,16 +1,16 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import * as framework from '@powersync/lib-services-framework';
 import { GetCheckpointChangesOptions, InternalOpId, storage } from '@powersync/service-core';
-import { PowerSyncMongo } from './db.js';
+import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js';
 
 export type MongoCheckpointAPIOptions = {
-  db: PowerSyncMongo;
+  db: VersionedPowerSyncMongo;
   mode: storage.WriteCheckpointMode;
   sync_rules_id: number;
 };
 
 export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
-  readonly db: PowerSyncMongo;
+  readonly db: VersionedPowerSyncMongo;
   private _mode: storage.WriteCheckpointMode;
 
   constructor(options: MongoCheckpointAPIOptions) {
@@ -166,7 +166,7 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
 }
 
 export async function batchCreateCustomWriteCheckpoints(
-  db: PowerSyncMongo,
+  db: VersionedPowerSyncMongo,
   session: mongo.ClientSession,
   checkpoints: storage.CustomWriteCheckpointOptions[],
   opId: InternalOpId

package/src/storage/implementation/OperationBatch.ts

@@ -2,6 +2,7 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 
 import { storage } from '@powersync/service-core';
+import { mongoTableId } from '../storage-index.js';
 
 /**
  * Maximum number of operations in a batch.
@@ -86,8 +87,8 @@ export class RecordOperation {
     const beforeId = record.beforeReplicaId ?? record.afterReplicaId;
     this.afterId = afterId;
     this.beforeId = beforeId;
-    this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId);
-    this.internalAfterKey = afterId ? cacheKey(record.sourceTable.id, afterId) : null;
+    this.internalBeforeKey = cacheKey(mongoTableId(record.sourceTable.id), beforeId);
+    this.internalAfterKey = afterId ? cacheKey(mongoTableId(record.sourceTable.id), afterId) : null;
 
     this.estimatedSize = estimateRowSize(record.before) + estimateRowSize(record.after);
   }

package/src/storage/implementation/PersistedBatch.ts

@@ -5,9 +5,9 @@ import * as bson from 'bson';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
 import { InternalOpId, storage, utils } from '@powersync/service-core';
-import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatch.js';
+import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatch.js';
 import { MongoIdSequence } from './MongoIdSequence.js';
-import { PowerSyncMongo } from './db.js';
+import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js';
 import {
   BucketDataDocument,
   BucketParameterDocument,
@@ -16,7 +16,7 @@ import {
   CurrentDataDocument,
   SourceKey
 } from './models.js';
-import { replicaIdToSubkey } from '../../utils/util.js';
+import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js';
 
 /**
  * Maximum size of operations we write in a single transaction.
@@ -63,6 +63,7 @@ export class PersistedBatch {
   currentSize = 0;
 
   constructor(
+    private db: VersionedPowerSyncMongo,
     private group_id: number,
     writtenSize: number,
     options?: { logger?: Logger }
@@ -132,7 +133,7 @@ export class PersistedBatch {
               o: op_id
             },
             op: 'PUT',
-            source_table: options.table.id,
+            source_table: mongoTableId(options.table.id),
             source_key: options.sourceKey,
             table: k.table,
             row_id: k.id,
@@ -159,7 +160,7 @@ export class PersistedBatch {
               o: op_id
             },
             op: 'REMOVE',
-            source_table: options.table.id,
+            source_table: mongoTableId(options.table.id),
             source_key: options.sourceKey,
             table: bd.table,
             row_id: bd.id,
@@ -208,7 +209,7 @@ export class PersistedBatch {
             _id: op_id,
             key: {
               g: this.group_id,
-              t: sourceTable.id,
+              t: mongoTableId(sourceTable.id),
               k: sourceKey
             },
             lookup: binLookup,
@@ -230,7 +231,7 @@ export class PersistedBatch {
             _id: op_id,
             key: {
               g: this.group_id,
-              t: sourceTable.id,
+              t: mongoTableId(sourceTable.id),
               k: sourceKey
             },
             lookup: lookup,
@@ -243,7 +244,7 @@ export class PersistedBatch {
     }
   }
 
-  deleteCurrentData(id: SourceKey) {
+  hardDeleteCurrentData(id: SourceKey) {
     const op: mongo.AnyBulkWriteOperation<CurrentDataDocument> = {
       deleteOne: {
         filter: { _id: id }
@@ -253,12 +254,41 @@ export class PersistedBatch {
     this.currentSize += 50;
   }
 
+  /**
+   * Mark a current_data document as soft deleted, to delete on the next commit.
+   *
+   * If softDeleteCurrentData is not enabled, this falls back to a hard delete.
+   */
+  softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint) {
+    if (!this.db.storageConfig.softDeleteCurrentData) {
+      this.hardDeleteCurrentData(id);
+      return;
+    }
+    const op: mongo.AnyBulkWriteOperation<CurrentDataDocument> = {
+      updateOne: {
+        filter: { _id: id },
+        update: {
+          $set: {
+            data: EMPTY_DATA,
+            buckets: [],
+            lookups: [],
+            pending_delete: checkpointGreaterThan
+          }
+        },
+        upsert: true
+      }
+    };
+    this.currentData.push(op);
+    this.currentSize += 50;
+  }
+
   upsertCurrentData(id: SourceKey, values: Partial<CurrentDataDocument>) {
     const op: mongo.AnyBulkWriteOperation<CurrentDataDocument> = {
       updateOne: {
         filter: { _id: id },
         update: {
-          $set: values
+          $set: values,
+          $unset: { pending_delete: 1 }
         },
         upsert: true
       }
@@ -276,7 +306,8 @@ export class PersistedBatch {
     );
   }
 
-  async flush(db: PowerSyncMongo, session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
+  async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
+    const db = this.db;
     const startAt = performance.now();
     let flushedSomething = false;
     if (this.bucketData.length > 0) {
@@ -297,7 +328,7 @@ export class PersistedBatch {
     }
     if (this.currentData.length > 0) {
       flushedSomething = true;
-      await db.current_data.bulkWrite(this.currentData, {
+      await db.common_current_data.bulkWrite(this.currentData, {
         session,
         // may update and delete data within the same batch - order matters
         ordered: true