@powersync/service-module-mongodb-storage 0.18.3 → 0.19.0

package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts

@@ -3,7 +3,7 @@ import { Logger, ReplicationAbortedError } from '@powersync/lib-services-framewo
 import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
 import { BucketDefinitionId, ParameterIndexId, SyncConfig } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
+import { clearCollectionInIdRanges, idPrefixFilter } from '../../../utils/util.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 import {
   BucketStateDocumentV3,
@@ -11,6 +11,8 @@ import {
   SourceTableDocumentV3,
   SyncConfigDefinition
 } from './models.js';
+import { ObjectStorage } from './object-storage/ObjectStorage.js';
+import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
 
 type SyncConfigState = ReplicationStreamDocumentV3['sync_configs'][number];
 
@@ -34,6 +36,8 @@ export interface MongoStoppedSyncConfigCleanupOptions extends storage.CleanupSto
   replicationStreamId: number;
   db: VersionedPowerSyncMongoV3;
   logger: Logger;
+  clearBatchThrottleRate: number;
+  objectStorage?: ObjectStorage;
 }
 
 export class MongoStoppedSyncConfigCleanup {
@@ -41,16 +45,20 @@ export class MongoStoppedSyncConfigCleanup {
   private readonly replicationStreamId: number;
   private readonly signal: AbortSignal | undefined;
   private readonly logger: Logger;
+  private readonly objectStorage: ObjectStorage | undefined;
   private readonly defaultSchema: string;
   private readonly sourceConnectionTag: string;
+  private readonly clearBatchThrottleRate: number;
 
   constructor(options: MongoStoppedSyncConfigCleanupOptions) {
     this.db = options.db;
     this.replicationStreamId = options.replicationStreamId;
     this.signal = options.signal;
     this.logger = options.logger;
+    this.objectStorage = options.objectStorage;
     this.defaultSchema = options.defaultSchema;
     this.sourceConnectionTag = options.sourceConnectionTag;
+    this.clearBatchThrottleRate = options.clearBatchThrottleRate;
   }
 
   async run(): Promise<storage.CleanupStoppedSyncConfigsResult> {
@@ -371,9 +379,13 @@ export class MongoStoppedSyncConfigCleanup {
   }
 
   private async dropBucketDataCollections(definitionIds: BucketDefinitionId[]): Promise<number> {
+    const lifecycle = this.objectStorage
+      ? new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage)
+      : null;
     for (const definitionId of definitionIds) {
       this.throwIfAborted();
       await this.dropCollection(this.db.bucketData(this.replicationStreamId, definitionId));
+      await lifecycle?.deletePrefix(lifecycle.definitionPrefix(definitionId), { signal: this.signal });
     }
     return definitionIds.length;
   }
@@ -393,26 +405,16 @@ export class MongoStoppedSyncConfigCleanup {
     let deletedCount = 0;
     for (const definitionId of definitionIds) {
       this.throwIfAborted();
-      const result = await retryOnMongoMaxTimeMSExpired(
-        () =>
-          collection.deleteMany(
-            {
-              _id: idPrefixFilter<BucketStateDocumentV3['_id']>({ d: definitionId }, ['b'])
-            },
-            { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
-          ),
+      deletedCount += await clearCollectionInIdRanges(
+        this.logger,
+        'bucket state',
+        collection,
         {
-          signal: this.signal,
-          abortMessage: 'Aborted stopped sync config cleanup',
-          retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5,
-          onRetry: () => {
-            this.logger.info(
-              `Cleared batch of bucket state in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...`
-            );
-          }
-        }
+          _id: idPrefixFilter<BucketStateDocumentV3['_id']>({ d: definitionId }, ['b'])
+        },
+        this.signal,
+        this.clearBatchThrottleRate
       );
-      deletedCount += result.deletedCount;
     }
     return deletedCount;
   }

package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts

@@ -24,7 +24,7 @@ import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
 import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
 import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
-import { loadBucketDataDocument } from './bucket-format.js';
+import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
 import {
   BucketDataDocumentV3,
   BucketParameterDocumentV3,
@@ -37,12 +37,16 @@ import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
 import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
 import { MongoCompactorV3 } from './MongoCompactorV3.js';
 import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
+import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
+import { ObjectStorage } from './object-storage/ObjectStorage.js';
+import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
 export interface MongoSyncBucketStorageContextV3 {
   db: VersionedPowerSyncMongoV3;
   replicationStreamId: number;
   readPreference?: mongo.ReadPreference;
+  objectStorage: ObjectStorage | undefined;
   /**
    * Persisted mapping of the single sync config that read operations are served from.
    *
@@ -52,47 +56,53 @@ export interface MongoSyncBucketStorageContextV3 {
   readonly mapping: SingleSyncConfigBucketDefinitionMapping;
 }
 
-function* walkDocumentOps(
-  data: BucketDataDoc[],
-  documentOpCounts: number[],
-  documentSizes: number[]
-): Generator<{ row: BucketDataDoc; docIndex: number; isLastOpInDocument: boolean }> {
-  let opIndex = 0;
-  for (const [docIndex, opCount] of documentOpCounts.entries()) {
-    for (let i = 0; i < opCount; i++) {
-      yield { row: data[opIndex++], docIndex, isLastOpInDocument: i === opCount - 1 };
+const BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES = 16 * 1024 * 1024;
+
+/**
+ * Keep the documents hydrated for one sync response within a bounded payload or slightly higher.
+ *
+ * This deserializes on-demand, so no deserialization is performed for discarded data.
+ */
+function cutBucketDataBatch(rawDocuments: Buffer[]): {
+  documents: BucketDataDocumentV3[];
+  wasCut: boolean;
+} {
+  let cumulativeBytes = 0;
+  let documents: BucketDataDocumentV3[] = [];
+  for (const raw of rawDocuments) {
+    const doc = bson.deserialize(raw, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3;
+    documents.push(doc);
+    cumulativeBytes += doc.size;
+    if (cumulativeBytes > BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES) {
+      return {
+        documents,
+        wasCut: documents.length < rawDocuments.length
+      };
     }
   }
+  return { documents, wasCut: false };
 }
 
 function extractRowsFromDocument(
   doc: BucketDataDocumentV3,
   context: { replicationStreamId: number; definitionId: string },
-  bucketMap: Map<string, InternalOpId>,
-  endOpId: InternalOpId,
-  remainingLimit: number
-): { rows: BucketDataDoc[]; remainingLimit: number; limitReached: boolean } {
+  bucketStart: InternalOpId,
+  endOpId: InternalOpId
+): BucketDataDoc[] {
   const rows: BucketDataDoc[] = [];
   for (const row of loadBucketDataDocument(context, doc)) {
-    const bucket = row.bucketKey.bucket;
-    const bucketStart = bucketMap.get(bucket);
-    if (bucketStart == null) {
-      throw new Error(`data for unexpected bucket: ${bucket}`);
-    }
+    // In theory a binary search could be faster than a linear scan to find the start.
+    // In practice, most cases should not filter out anything here.
     if (row.o <= bucketStart) {
       continue;
     }
     if (row.o > endOpId) {
-      continue;
+      break;
     }
 
     rows.push(row);
-    remainingLimit--;
-    if (remainingLimit <= 0) {
-      return { rows, remainingLimit, limitReached: true };
-    }
   }
-  return { rows, remainingLimit, limitReached: false };
+  return rows;
 }
 
 export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
@@ -337,6 +347,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     const self = this;
     return {
       db: this.db,
+      objectStorage: this.objectStorage,
       replicationStreamId: this.replicationStreamId,
       readPreference: this.readPreference,
       get mapping() {
@@ -357,14 +368,27 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     checkpoint: MongoSyncBucketStorageCheckpoint,
     dataBuckets: storage.BucketDataRequest[],
     options?: storage.BucketDataBatchOptions
-  ): AsyncIterable<storage.SyncBucketDataChunk> {
+  ): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
     return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options);
   }
 
-  protected async clearBucketData(_signal?: AbortSignal): Promise<void> {
+  protected async clearBucketData(signal?: AbortSignal): Promise<void> {
     for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
       await collection.drop();
     }
+    if (this.objectStorage) {
+      const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
+      await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
+    }
+    await this.db
+      .pendingObjectStorageDeletes(this.replicationStreamId)
+      .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
+      .catch((error) => {
+        if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+          return;
+        }
+        throw error;
+      });
   }
 
   protected async clearParameterIndexes(_signal?: AbortSignal): Promise<void> {
@@ -412,7 +436,9 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
       signal: options.signal,
       logger: options.logger ?? this.logger,
       defaultSchema: options.defaultSchema,
-      sourceConnectionTag: options.sourceConnectionTag
+      sourceConnectionTag: options.sourceConnectionTag,
+      objectStorage: this.objectStorage,
+      clearBatchThrottleRate: this.clearBatchThrottleRate
     }).run();
   }
 
@@ -534,7 +560,7 @@ export async function* getBucketDataBatchV3(
   checkpoint: MongoSyncBucketStorageCheckpoint,
   dataBuckets: storage.BucketDataRequest[],
   options?: storage.BucketDataBatchOptions
-): AsyncIterable<storage.SyncBucketDataChunk> {
+): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
   if (dataBuckets.length == 0) {
     return;
   }
@@ -559,8 +585,9 @@ export async function* getBucketDataBatchV3(
   const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
   const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
   const end = checkpoint.checkpoint;
-  let remainingLimit = batchLimit;
 
+  // Group requests by definition, so that we can query each definition's bucket data in a single query.
+  // We only return the results of a single query per batch.
   const requestsByDefinition = new Map<string, storage.BucketDataRequest[]>();
   for (const request of dataBuckets) {
     const definitionId = ctx.mapping.bucketSourceId(request.source);
@@ -571,9 +598,6 @@ export async function* getBucketDataBatchV3(
 
   const definitionGroups = Array.from(requestsByDefinition.entries());
   for (const [groupIndex, [definitionId, requests]] of definitionGroups.entries()) {
-    if (remainingLimit <= 0) {
-      break;
-    }
     const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1;
     const bucketMap = new Map(requests.map((request) => [request.bucket, request.start]));
     const filters = Array.from(bucketMap.entries()).map(([bucket, start]) => ({
@@ -589,9 +613,8 @@ export async function* getBucketDataBatchV3(
     // MongoDB Filter<T> doesn't accept the $or operator in its type.
     const filter = { $or: filters } as unknown as mongo.Filter<BucketDataDocumentV3>;
     const context = { replicationStreamId: ctx.replicationStreamId, definitionId };
-    const limit = remainingLimit;
 
-    const cursorOptions = { limit: remainingLimit, batchSize: remainingLimit + 1 };
+    const cursorOptions = { limit: batchLimit, batchSize: batchLimit + 1 };
 
     // raw: true returns Buffers, but the driver typing doesn't reflect that
     // without an explicit cast to FindCursor<Buffer>.
@@ -609,88 +632,44 @@ export async function* getBucketDataBatchV3(
       throw lib_mongo.mapQueryError(e, 'while reading bucket data');
     });
 
-    if (cursorOptions.limit != null && rawData.length >= cursorOptions.limit) {
+    if (rawData.length >= cursorOptions.limit) {
       hasMore = true;
     }
 
-    const data: BucketDataDoc[] = [];
-    const documentOpCounts: number[] = [];
-    const documentSizes: number[] = [];
-    let sharedRemainingLimit = limit;
-    let limitReached = false;
-    // Buckets whose matched document contributed no rows after filtering.
-    const completeEmptyBuckets = new Set<string>();
-
-    for (const raw of rawData) {
-      const doc = bson.deserialize(raw, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3;
-      const {
-        rows,
-        remainingLimit,
-        limitReached: docLimitReached
-      } = extractRowsFromDocument(doc, context, bucketMap, end, sharedRemainingLimit);
-      if (rows.length == 0) {
-        // The document straddles the requested (start, end] window: it matched the
-        // query, but none of its ops are in range. Since its _id.o (max op) must be
-        // > end (any op <= end would have been > start, and thus in range), and
-        // document ranges per bucket are disjoint, no later document for this bucket
-        // can match either. The bucket is complete through the checkpoint.
-        completeEmptyBuckets.add(doc._id.b);
-      }
-      data.push(...rows);
-      documentOpCounts.push(rows.length);
-      documentSizes.push(raw.byteLength);
-      sharedRemainingLimit = remainingLimit;
-      if (docLimitReached) {
-        limitReached = true;
-        break;
-      }
-    }
-
-    const batchHasMore = hasMore || limitReached;
-
-    // Empty chunks are not forwarded to clients, but report progress to the caller:
-    // the bucket's position advances to the checkpoint, so it is not re-requested.
-    // If the batch produced no data at all, the last empty chunk also carries the
-    // has_more signal, so the caller re-requests the remaining buckets instead of
-    // treating an all-filtered batch as the end of the stream.
-    const emptyBuckets = Array.from(completeEmptyBuckets);
-    for (const [index, bucket] of emptyBuckets.entries()) {
-      const startOpId = bucketMap.get(bucket);
-      if (startOpId == null) {
-        throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
-      }
-      const isLastChunkOfBatch = data.length == 0 && index == emptyBuckets.length - 1;
-      yield {
-        chunkData: {
-          bucket,
-          after: internalToExternalOpId(startOpId),
-          has_more: isLastChunkOfBatch && batchHasMore,
-          data: [],
-          next_after: internalToExternalOpId(end)
-        },
-        targetOp: null
-      };
-    }
-
-    if (data.length == 0) {
-      if (batchHasMore) {
-        // The remaining documents are read in the next round, after the caller has
-        // advanced the positions of the empty buckets above.
-        return;
-      }
-      continue;
+    // Deserialize the raw documents and cut the batch to a bounded size. Any data not
+    // making the cut will be read in the next round, using a fresh query.
+    const cutBatch = cutBucketDataBatch(rawData);
+    const docs = cutBatch.documents;
+    if (cutBatch.wasCut) {
+      hasMore = true;
     }
 
-    remainingLimit -= data.length;
+    // Hydrate any operations from object storage.
+    // In the future we can do this in a more pipelined fashion, but for now we hydrate
+    // the entire batch at once.
+    await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal });
 
     let currentChunkSizeBytes = 0;
     let currentChunk: utils.SyncBucketData | null = null;
     let targetOp: InternalOpId | null = null;
+    let seenBuckets = new Set<string>();
+    const batchHasMore = hasMore;
+
+    for (const doc of docs) {
+      const bucket = doc._id.b;
+      seenBuckets.add(bucket);
+      const bucketStart = bucketMap.get(bucket);
+      if (bucketStart == null) {
+        throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
+      }
 
-    for (const { row, docIndex, isLastOpInDocument } of walkDocumentOps(data, documentOpCounts, documentSizes)) {
-      const bucket = row.bucketKey.bucket;
-
-      if (currentChunk == null || currentChunk.bucket != bucket || currentChunkSizeBytes >= chunkSizeLimitBytes) {
+      // Reached a new bucket or size limit: yield the current chunk and start a new one.
+      if (
+        currentChunk == null ||
+        currentChunk.bucket != bucket ||
+        currentChunkSizeBytes >= chunkSizeLimitBytes ||
+        currentChunk.data.length >= batchLimit
+      ) {
         let start: ProtocolOpId | undefined = undefined;
         if (currentChunk != null) {
           if (currentChunk.bucket == bucket) {
@@ -706,12 +685,9 @@ export async function* getBucketDataBatchV3(
         }
 
         if (start == null) {
-          const startOpId = bucketMap.get(bucket);
-          if (startOpId == null) {
-            throw new Error(`data for unexpected bucket: ${bucket}`);
-          }
-          start = internalToExternalOpId(startOpId);
+          start = internalToExternalOpId(bucketStart);
         }
+
         currentChunk = {
           bucket,
           after: start,
@@ -721,27 +697,47 @@ export async function* getBucketDataBatchV3(
         };
       }
 
-      const entry = mapOpEntry(row);
-      if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) {
-        targetOp = row.target_op;
-      }
-
-      currentChunk.data.push(entry);
-      currentChunk.next_after = entry.op_id;
-
-      if (isLastOpInDocument) {
-        currentChunkSizeBytes += documentSizes[docIndex];
-      }
+      const rows = extractRowsFromDocument(doc, context, bucketStart, end);
+      currentChunk.data.push(...rows.map(mapOpEntry));
+      currentChunk.next_after = currentChunk.data.at(-1)?.op_id ?? internalToExternalOpId(end);
+      targetOp = maxOpId(targetOp, doc.target_op);
+      currentChunkSizeBytes += doc.size;
     }
 
     if (currentChunk != null) {
       const yieldChunk = currentChunk;
-      yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups);
+      // The last chunk may contain more data that was cut in this batch.
+      yieldChunk.has_more = batchHasMore;
       yield { chunkData: yieldChunk, targetOp };
     }
 
-    if (batchHasMore || remainingLimit <= 0) {
-      return;
+    if (!batchHasMore) {
+      for (const bucket of bucketMap.keys()) {
+        if (!seenBuckets.has(bucket)) {
+          // We processed everything for this definition group, but this bucket had no data in the batch.
+          // Yield an empty chunk to indicate that it is complete.
+          // This prevents re-querying the same bucket in the next batch.
+          yield {
+            chunkData: {
+              bucket,
+              after: internalToExternalOpId(bucketMap.get(bucket)!),
+              has_more: false,
+              data: [],
+              next_after: internalToExternalOpId(end)
+            },
+            targetOp
+          };
+        }
+      }
+    }
+
+    if (currentChunk != null) {
+      // We yielded data in this group (aside from empty buckets).
+      // Return to the caller to allow them to process it before continuing to the next group.
+      yield { hasMore: batchHasMore || hasLaterDefinitionGroups };
+      break;
+    } else {
+      // No data in this definition group - continue in the next group.
     }
   }
 }

package/src/storage/implementation/v3/PersistedBatchV3.ts

@@ -1,6 +1,6 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { InternalOpId, storage } from '@powersync/service-core';
+import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
 import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { mongoTableId } from '../../../utils/util.js';
@@ -8,6 +8,7 @@ import { BucketDataDoc } from '../common/BucketDataDoc.js';
 import {
   BucketStateUpdate,
   PersistedBatch,
+  PersistedBatchOptions,
   SaveParameterDataOptions,
   UpsertCurrentDataOptions
 } from '../common/PersistedBatch.js';
@@ -22,14 +23,29 @@ import {
   SourceTableDocumentV3,
   taggedBucketParameterDocumentToTagged
 } from './models.js';
+import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
 export class PersistedBatchV3 extends PersistedBatch {
   currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
   sourceTablePendingDeletes = new Map<string, InternalOpId>();
+  protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
 
   declare protected readonly db: VersionedPowerSyncMongoV3;
 
+  constructor(
+    db: VersionedPowerSyncMongoV3,
+    group_id: number,
+    mapping: BucketDefinitionMapping,
+    writtenSize: number,
+    options?: PersistedBatchOptions
+  ) {
+    super(db, group_id, mapping, writtenSize, options);
+    if (this.objectStorage) {
+      this.objectStorageLifecycle = new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage);
+    }
+  }
+
   // Abstract override from PersistedBatch (V3-specific error message)
 
   protected override checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
@@ -209,26 +225,63 @@ export class PersistedBatchV3 extends PersistedBatch {
       operationsByDefinition.set(document.bucketKey.definitionId, existing);
     }
 
-    for (const [definitionId, documents] of operationsByDefinition.entries()) {
-      const operationsByBucket = new Map<string, BucketDataDoc[]>();
-      for (const document of documents) {
-        const existing = operationsByBucket.get(document.bucketKey.bucket) ?? [];
-        existing.push(document);
-        operationsByBucket.set(document.bucketKey.bucket, existing);
-      }
+    let uploadCount = 0;
+    const plans = Array.from(operationsByDefinition, ([definitionId, documents]) => {
+      const operationsByBucket = Map.groupBy(documents, (document) => document.bucketKey.bucket);
+      const lifecycle = this.objectStorageLifecycle;
+      const createInserts: (() => Promise<mongo.AnyBulkWriteOperation<BucketDataDocumentV3>>)[] = [];
+
+      for (const [bucket, ops] of operationsByBucket) {
+        for (const chunk of chunkBucketData(ops)) {
+          const serialized = serializeBucketData(bucket, chunk);
+          if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) {
+            createInserts.push(async () => ({
+              insertOne: {
+                document: serialized
+              }
+            }));
+            continue;
+          }
 
-      const inserts: mongo.AnyBulkWriteOperation<BucketDataDocumentV3>[] = [];
-      for (const [bucket, ops] of operationsByBucket.entries()) {
-        const chunks = chunkBucketData(ops);
-        for (const chunk of chunks) {
-          inserts.push({
-            insertOne: {
-              document: serializeBucketData(bucket, chunk)
-            }
+          uploadCount += 1;
+          createInserts.push(async () => {
+            const minOp = chunk[0].o;
+            const maxOp = chunk[chunk.length - 1].o;
+            const { ops: bucketOps, ...metadata } = serialized;
+            const path = lifecycle.allocatePath(definitionId, bucket, minOp, maxOp);
+            const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!);
+            return {
+              insertOne: {
+                document: {
+                  ...metadata,
+                  storage_ref: {
+                    path,
+                    file_size: fileSize
+                  }
+                }
+              }
+            };
           });
         }
       }
 
+      return { definitionId, createInserts };
+    });
+
+    const createAllInserts = () =>
+      Promise.all(
+        plans.map(async ({ definitionId, createInserts }) => ({
+          definitionId,
+          inserts: await Promise.all(createInserts.map((createInsert) => createInsert()))
+        }))
+      );
+
+    // S3ObjectStorage applies one shared concurrency limit across all callers,
+    // so replication can schedule its uploads together without creating a
+    // separate limiter here.
+    const writes = await createAllInserts();
+
+    for (const { definitionId, inserts } of writes) {
       if (inserts.length > 0) {
         await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
           session,

package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts

@@ -7,6 +7,7 @@ import {
   BucketParameterDocumentV3,
   BucketStateDocumentV3,
   CurrentDataDocumentV3,
+  ObjectStorageDeletionMarker,
   SourceTableDocumentV3,
   SyncConfigDefinition
 } from './models.js';
@@ -69,6 +70,10 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
     return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
   }
 
+  pendingObjectStorageDeletes(replicationStreamId: number): mongo.Collection<ObjectStorageDeletionMarker> {
+    return this.db.collection<ObjectStorageDeletionMarker>(`pending_object_storage_deletes_${replicationStreamId}`);
+  }
+
   listBucketDataCollections(replicationStreamId: number) {
     return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
   }