@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js

@@ -1,38 +1,77 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
-import { internalToExternalOpId, maxLsn, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
+import { internalToExternalOpId, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
 import * as bson from 'bson';
 import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
-import { MongoPersistedSyncRulesContentV3 } from '../MongoPersistedSyncRulesContent.js';
 import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js';
-import { loadBucketDataDocumentV3 } from './models.js';
+import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
+import { deserializeParameterLookup, serializeParameterLookup } from './models.js';
 import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
 import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
 import { MongoCompactorV3 } from './MongoCompactorV3.js';
 import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js';
-import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
+import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
+import { MongoWriteCheckpointAPIV3 } from './MongoWriteCheckpointAPIV3.js';
+import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
+import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
+import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
+const BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES = 16 * 1024 * 1024;
+/**
+ * Keep the documents hydrated for one sync response within a bounded payload or slightly higher.
+ *
+ * This deserializes on-demand, so no deserialization is performed for discarded data.
+ */
+function cutBucketDataBatch(rawDocuments) {
+    let cumulativeBytes = 0;
+    let documents = [];
+    for (const raw of rawDocuments) {
+        const doc = bson.deserialize(raw, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
+        documents.push(doc);
+        cumulativeBytes += doc.size;
+        if (cumulativeBytes > BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES) {
+            return {
+                documents,
+                wasCut: documents.length < rawDocuments.length
+            };
+        }
+    }
+    return { documents, wasCut: false };
+}
+function extractRowsFromDocument(doc, context, bucketStart, endOpId) {
+    const rows = [];
+    for (const row of loadBucketDataDocument(context, doc)) {
+        // In theory a binary search could be faster than a linear scan to find the start.
+        // In practice, most cases should not filter out anything here.
+        if (row.o <= bucketStart) {
+            continue;
+        }
+        if (row.o > endOpId) {
+            break;
+        }
+        rows.push(row);
+    }
+    return rows;
+}
 export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
-    syncRulesV3;
-    constructor(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options) {
-        super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
-        if (!(sync_rules instanceof MongoPersistedSyncRulesContentV3)) {
+    constructor(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
+        super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
+        if (replicationStream.syncConfigIds.length == 0) {
             throw new ServiceAssertionError('Missing sync config id for storage v3');
         }
-        this.syncRulesV3 = sync_rules;
     }
-    get syncConfigId() {
-        return this.syncRulesV3.syncConfigId;
+    get syncConfigIds() {
+        return this.replicationStream.syncConfigIds;
     }
     get syncRulesCollection() {
         return this.db.sync_rules;
     }
     syncConfigMatch(extra = {}) {
         return {
-            _id: this.group_id,
+            _id: this.replicationStreamId,
             sync_configs: {
                 $elemMatch: {
-                    _id: this.syncConfigId,
+                    _id: { $in: this.syncConfigIds },
                     ...extra
                 }
             }
@@ -41,39 +80,25 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     syncConfigProjection(extra = {}) {
         return {
             ...extra,
-            sync_configs: {
-                $elemMatch: {
-                    _id: this.syncConfigId
-                }
-            }
+            sync_configs: 1
         };
     }
-    syncConfigArrayFilters() {
-        return [{ 'config._id': this.syncConfigId }];
-    }
-    /**
-     * For now, we only support a single sync config per replication stream.
-     *
-     * In the future we'll add support for multiple.
-     */
-    selectedSyncConfig(doc) {
-        return doc?.sync_configs?.[0] ?? null;
+    selectedSyncConfigs(doc) {
+        return doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
     }
     async initializeVersionStorage() {
-        const mapping = this.mapping;
-        for (let source of mapping.allBucketDefinitionIds()) {
-            const collection = this.db.bucketDataV3(this.group_id, source).collectionName;
-            await this.db.db
-                .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } })
-                .catch((error) => {
+        const storageIds = this.storageIds;
+        for (const source of storageIds.bucketDefinitionIds) {
+            const collection = this.db.bucketData(this.replicationStreamId, source).collectionName;
+            await this.db.db.createCollection(collection, {}).catch((error) => {
                 if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') {
                     return;
                 }
                 throw error;
             });
         }
-        for (let indexId of mapping.allParameterIndexIds()) {
-            await this.db.parameterIndexV3(this.group_id, indexId).createIndex({
+        for (const indexId of storageIds.parameterIndexIds) {
+            await this.db.parameterIndex(this.replicationStreamId, indexId).createIndex({
                 lookup: 1,
                 key: 1,
                 _id: -1
@@ -83,53 +108,85 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
         }
     }
     createMongoChecksums(options) {
-        return new MongoChecksumsV3(this.db, this.group_id, {
+        return new MongoChecksumsV3(this.db, this.replicationStreamId, {
             ...options.checksumOptions,
+            checksumCacheTtlMs: options.checksumCacheTtlMs,
             storageConfig: options?.storageConfig,
-            mapping: this.sync_rules.mapping
+            syncConfigMapping: () => this.singleSyncConfigMapping()
+        });
+    }
+    createWriteCheckpointAPI(options) {
+        return new MongoWriteCheckpointAPIV3({
+            ...options,
+            db: this.db,
+            syncConfigMapping: () => this.singleSyncConfigMapping()
         });
     }
     createMongoCompactor(options) {
         return new MongoCompactorV3(this, this.db, options);
     }
-    createMongoParameterCompactor(checkpoint, options) {
-        return new MongoParameterCompactorV3(this.db, this.group_id, checkpoint, options);
+    async compactInitialReplication(options) {
+        this.logger.info(`Compacting chunks after initial replication...`);
+        const start = Date.now();
+        const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
+        const compactedBuckets = await this.createMongoCompactor({
+            ...options,
+            maxOpId,
+            compactChunksOnly: true,
+            logger: this.logger
+        }).compact();
+        this.logger.info(`Compacted chunks after initial replication in ${(Date.now() - start) / 1000}s`);
+        return { buckets: compactedBuckets };
     }
-    createWriterImpl(batchOptions) {
-        return new MongoBucketBatchV3(batchOptions);
+    createMongoParameterCompactor(checkpoint, options) {
+        return new MongoParameterCompactorV3(this.db, this.replicationStreamId, checkpoint, options);
+    }
+    async fetchPersistedOpHead() {
+        const doc = await this.syncRulesCollection.findOne({ _id: this.replicationStreamId }, { projection: { last_persisted_op: 1 } });
+        return doc?.last_persisted_op == null ? null : BigInt(doc.last_persisted_op);
+    }
+    async createWriterImpl(options) {
+        const batchOptions = this.writerBatchOptions(options);
+        const doc = await this.syncRulesCollection.findOne({ _id: this.replicationStreamId }, { projection: { resume_lsn: 1 } });
+        return new MongoBucketBatchV3({
+            ...batchOptions,
+            // The stream-level replication position - per-config checkpoint LSNs are consistency
+            // markers and do not affect where replication resumes.
+            resumeFromLsn: doc?.resume_lsn ?? null,
+            syncConfigIds: this.syncConfigIds
+        });
     }
     async fetchCheckpointState(session) {
         const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch({
             state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
         }), {
             session,
-            projection: this.syncConfigProjection()
+            // The invalidation fence must be read in the same snapshot as the checkpoint.
+            projection: this.syncConfigProjection({ 'parameter_compaction.checkpoint_changes_invalid_before': 1 })
         });
-        const syncConfig = this.selectedSyncConfig(doc);
-        if (!syncConfig?.snapshot_done) {
+        // Checkpoints are served from the single active config. A PROCESSING config in the same
+        // stream (incremental reprocessing) does not affect checkpoints until it is activated.
+        const syncConfigs = this.selectedSyncConfigs(doc).filter((config) => [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED].includes(config.state));
+        if (syncConfigs.length > 1) {
+            // Activation atomically replaces the previous active config, so this cannot happen unless
+            // the stored state is corrupt.
+            throw new ServiceAssertionError(`Expected a single active sync config, got ${syncConfigs.map((config) => config._id.toHexString()).join(', ')}`);
+        }
+        const syncConfig = syncConfigs[0];
+        if (syncConfig == null || !syncConfig.snapshot_done) {
             return null;
         }
         return {
             checkpoint: syncConfig.last_checkpoint ?? 0n,
-            lsn: syncConfig.last_checkpoint_lsn ?? null
-        };
-    }
-    async getWriterSyncState() {
-        const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
-            projection: this.syncConfigProjection({ snapshot_lsn: 1 })
-        });
-        const syncConfig = this.selectedSyncConfig(doc);
-        const checkpointLsn = syncConfig?.last_checkpoint_lsn ?? null;
-        return {
-            lastCheckpointLsn: checkpointLsn,
-            resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
-            keepaliveOp: syncConfig?.keepalive_op ?? null,
-            syncConfigId: this.syncConfigId
+            lsn: syncConfig.last_checkpoint_lsn ?? null,
+            // Stream-level state: shared by all sync configs. Defaults to 0n for streams that have
+            // never been compacted.
+            parameterChangesInvalidBefore: doc?.parameter_compaction?.checkpoint_changes_invalid_before ?? 0n
         };
     }
     async terminateSyncRuleState() {
         await this.db.sync_rules.updateOne({
-            _id: this.group_id
+            _id: this.replicationStreamId
         }, {
             $set: {
                 state: storage.SyncRuleState.TERMINATED,
@@ -140,43 +197,70 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     }
     async getStatusImpl() {
         const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
-            projection: this.syncConfigProjection({ state: 1, snapshot_lsn: 1, keepalive_op: 1 })
+            projection: this.syncConfigProjection({ resume_lsn: 1 })
         });
-        const syncConfig = this.selectedSyncConfig(doc);
-        if (doc == null || syncConfig == null) {
+        const syncConfigs = this.selectedSyncConfigs(doc);
+        if (doc == null || syncConfigs.length == 0) {
             throw new ServiceAssertionError('Cannot find replication stream status');
         }
         return {
-            snapshot_done: syncConfig.snapshot_done ?? false,
-            snapshot_lsn: doc.snapshot_lsn ?? null,
-            active: doc.state == storage.SyncRuleState.ACTIVE && syncConfig.state == storage.SyncRuleState.ACTIVE,
-            checkpoint_lsn: syncConfig.last_checkpoint_lsn ?? null,
-            keepalive_op: syncConfig.keepalive_op ?? null
+            snapshotDone: syncConfigs.every((config) => config.snapshot_done ?? false) &&
+                syncConfigs.every((config) => config.last_checkpoint_lsn != null),
+            resumeLsn: doc.resume_lsn ?? null
         };
     }
     async clearSyncRuleState() {
-        await this.syncRulesCollection.updateOne(this.syncConfigMatch(), {
+        // Clearing resets the entire replication stream (bucket data and the op sequence), so reset
+        // the checkpoint state for _all_ embedded sync configs, not only the ones selected for this
+        // storage instance. This maintains the invariant that no config has a last_checkpoint past
+        // the stream-level last_persisted_op.
+        await this.syncRulesCollection.updateOne({ _id: this.replicationStreamId }, {
             $set: {
                 persisted_lsn: null,
-                'sync_configs.$[config].snapshot_done': false,
-                'sync_configs.$[config].last_checkpoint_lsn': null,
-                'sync_configs.$[config].last_checkpoint': null,
-                'sync_configs.$[config].no_checkpoint_before': null,
-                'sync_configs.$[config].keepalive_op': null
+                'sync_configs.$[].snapshot_done': false,
+                'sync_configs.$[].last_checkpoint_lsn': null,
+                'sync_configs.$[].last_checkpoint': null,
+                'sync_configs.$[].no_checkpoint_before': null
             },
             $unset: {
-                snapshot_lsn: 1
+                resume_lsn: 1,
+                last_persisted_op: 1,
+                parameter_compaction: 1
             }
         }, {
-            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
-            arrayFilters: this.syncConfigArrayFilters()
+            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
         });
     }
+    /**
+     * The persisted mapping of the single sync config that read operations are served from.
+     *
+     * Reads always operate on a single sync config: read-path storage instances are
+     * constructed for the active sync config only (see MongoBucketStorage.getActiveSyncConfig),
+     * and checkpoints are served from the single active config (see fetchCheckpointState).
+     * Within a single sync config, unique names are the persistence key of its rule mapping,
+     * so its name-keyed {@link SingleSyncConfigBucketDefinitionMapping} resolves sources from
+     * any parse of that config unambiguously - no parsed-set identity is required.
+     *
+     * Throws on storage instances with multiple sync configs (replication-side instances),
+     * which must not serve reads.
+     */
+    singleSyncConfigMapping() {
+        const content = this.replicationStream.syncConfigContent;
+        if (content.length != 1) {
+            throw new ServiceAssertionError(`Read operations require a storage instance with a single sync config, got ${content.length}`);
+        }
+        return content[0].mapping;
+    }
     get versionContext() {
+        const self = this;
         return {
             db: this.db,
-            group_id: this.group_id,
-            mapping: this.mapping
+            objectStorage: this.objectStorage,
+            replicationStreamId: this.replicationStreamId,
+            readPreference: this.readPreference,
+            get mapping() {
+                return self.singleSyncConfigMapping();
+            }
         };
     }
     getParameterSetsImpl(checkpoint, lookups, limit) {
@@ -185,27 +269,42 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     getBucketDataBatchImpl(checkpoint, dataBuckets, options) {
         return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options);
     }
-    async clearBucketData(_signal) {
-        for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) {
+    async clearBucketData(signal) {
+        for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
             await collection.drop();
         }
+        const usage = new ObjectStorageUsage(this.db, this.replicationStreamId);
+        await this.db.client.withSession((session) => session.withTransaction(() => usage.removeStream(session)));
+        if (this.objectStorage) {
+            const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
+            await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
+        }
+        await this.db
+            .pendingObjectStorageDeletes(this.replicationStreamId)
+            .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
+            .catch((error) => {
+            if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
+                return;
+            }
+            throw error;
+        });
     }
     async clearParameterIndexes(_signal) {
-        for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) {
+        for (const collection of await this.db.listParameterIndexCollections(this.replicationStreamId)) {
             await collection.collection.drop();
         }
     }
     async clearSourceRecords(_signal) {
-        for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) {
+        for (const collection of await this.db.listSourceRecordCollections(this.replicationStreamId)) {
             await collection.drop();
         }
     }
     async clearBucketState(_signal) {
         await this.db
-            .bucketStateV3(this.group_id)
+            .bucketState(this.replicationStreamId)
             .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
             .catch((error) => {
-            if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+            if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
                 return;
             }
             throw error;
@@ -213,15 +312,38 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     }
     async clearSourceTables(_signal) {
         await this.db
-            .sourceTablesV3(this.group_id)
+            .sourceTables(this.replicationStreamId)
             .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
             .catch((error) => {
-            if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+            if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
                 return;
             }
             throw error;
         });
     }
+    async clearCustomCheckpointRequests(signal) {
+        for (const collection of await this.db.listCustomCheckpointRequestCollections(this.replicationStreamId)) {
+            signal?.throwIfAborted();
+            await collection.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }).catch((error) => {
+                if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+                    return;
+                }
+                throw error;
+            });
+        }
+    }
+    async cleanupStoppedSyncConfigs(options) {
+        return new MongoStoppedSyncConfigCleanup({
+            db: this.db,
+            replicationStreamId: this.replicationStreamId,
+            signal: options.signal,
+            logger: options.logger ?? this.logger,
+            defaultSchema: options.defaultSchema,
+            sourceConnectionTag: options.sourceConnectionTag,
+            objectStorage: this.objectStorage,
+            clearBatchThrottleRate: this.clearBatchThrottleRate
+        }).run();
+    }
     getDataBucketChangesImpl(options) {
         return getDataBucketChangesV3(this.versionContext, options);
     }
@@ -234,8 +356,8 @@ export async function getParameterSetsV3(ctx, checkpoint, lookups, limit) {
         setSessionSnapshotTime(session, checkpoint.snapshotTime);
         const buildLookupPipeline = (lookup, index) => {
             const indexId = lookup.indexId;
-            const collection = ctx.db.parameterIndexV3(ctx.group_id, indexId);
-            const lookupFilter = serializeParameterLookupV3(lookup);
+            const collection = ctx.db.parameterIndex(ctx.replicationStreamId, indexId);
+            const lookupFilter = serializeParameterLookup(lookup);
             return {
                 collection,
                 pipeline: [
@@ -313,13 +435,24 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
     if (dataBuckets.length == 0) {
         return;
     }
-    if (checkpoint == null) {
+    if (checkpoint.checkpoint == null) {
         throw new Error('checkpoint is null');
     }
+    const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
+    const readConcern = ctx.readPreference == null ? undefined : 'majority';
+    const session = readPreference == null || checkpoint.snapshotTime == null
+        ? undefined
+        : ctx.db.client.startSession({ causalConsistency: true });
+    await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
+    if (session != null) {
+        session.advanceOperationTime(checkpoint.snapshotTime);
+        session.advanceClusterTime(checkpoint.clusterTime);
+    }
     const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
     const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
-    const end = checkpoint;
-    let remainingLimit = batchLimit;
+    const end = checkpoint.checkpoint;
+    // Group requests by definition, so that we can query each definition's bucket data in a single query.
+    // We only return the results of a single query per batch.
     const requestsByDefinition = new Map();
     for (const request of dataBuckets) {
         const definitionId = ctx.mapping.bucketSourceId(request.source);
@@ -328,49 +461,67 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
         requestsByDefinition.set(definitionId, requests);
     }
     const definitionGroups = Array.from(requestsByDefinition.entries());
-    for (let groupIndex = 0; groupIndex < definitionGroups.length && remainingLimit > 0; groupIndex++) {
-        const [definitionId, requests] = definitionGroups[groupIndex];
+    for (const [groupIndex, [definitionId, requests]] of definitionGroups.entries()) {
         const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1;
         const bucketMap = new Map(requests.map((request) => [request.bucket, request.start]));
         const filters = Array.from(bucketMap.entries()).map(([bucket, start]) => ({
             _id: {
-                $gt: {
-                    b: bucket,
-                    o: start
-                },
-                $lte: {
-                    b: bucket,
-                    o: end
-                }
-            }
+                $gt: { b: bucket, o: start },
+                $lte: { b: bucket, o: new bson.MaxKey() }
+            },
+            min_op: { $lte: end }
+            // MongoDB Filter<T> doesn't accept compound _id ranges or dotted field paths in its type.
         }));
-        const cursor = ctx.db.bucketDataV3(ctx.group_id, definitionId).find({
-            $or: filters
-        }, {
-            session: undefined,
+        const collection = ctx.db.bucketData(ctx.replicationStreamId, definitionId);
+        // MongoDB Filter<T> doesn't accept the $or operator in its type.
+        const filter = { $or: filters };
+        const context = { replicationStreamId: ctx.replicationStreamId, definitionId };
+        const cursorOptions = { limit: batchLimit, batchSize: batchLimit + 1 };
+        // raw: true returns Buffers, but the driver typing doesn't reflect that
+        // without an explicit cast to FindCursor<Buffer>.
+        const cursor = collection.find(filter, {
+            session,
+            readPreference,
+            readConcern,
             sort: { _id: 1 },
-            limit: remainingLimit,
-            batchSize: remainingLimit + 1,
             raw: true,
-            maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
+            maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS,
+            ...cursorOptions
         });
-        let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => {
+        let { data: rawData, hasMore } = await readSingleBatch(cursor).catch((e) => {
             throw lib_mongo.mapQueryError(e, 'while reading bucket data');
         });
-        if (data.length == remainingLimit) {
-            batchHasMore = true;
+        if (rawData.length >= cursorOptions.limit) {
+            hasMore = true;
         }
-        if (data.length == 0) {
-            continue;
+        // Deserialize the raw documents and cut the batch to a bounded size. Any data not
+        // making the cut will be read in the next round, using a fresh query.
+        const cutBatch = cutBucketDataBatch(rawData);
+        const docs = cutBatch.documents;
+        if (cutBatch.wasCut) {
+            hasMore = true;
         }
-        remainingLimit -= data.length;
-        let chunkSizeBytes = 0;
+        // Hydrate any operations from object storage.
+        // In the future we can do this in a more pipelined fashion, but for now we hydrate
+        // the entire batch at once.
+        await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal, tracer: options?.tracer });
+        let currentChunkSizeBytes = 0;
         let currentChunk = null;
         let targetOp = null;
-        for (let rawData of data) {
-            const row = loadBucketDataDocumentV3({ replicationStreamId: ctx.group_id, definitionId }, bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS));
-            const bucket = row.bucketKey.bucket;
-            if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) {
+        let seenBuckets = new Set();
+        const batchHasMore = hasMore;
+        for (const doc of docs) {
+            const bucket = doc._id.b;
+            seenBuckets.add(bucket);
+            const bucketStart = bucketMap.get(bucket);
+            if (bucketStart == null) {
+                throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
+            }
+            // Reached a new bucket or size limit: yield the current chunk and start a new one.
+            if (currentChunk == null ||
+                currentChunk.bucket != bucket ||
+                currentChunkSizeBytes >= chunkSizeLimitBytes ||
+                currentChunk.data.length >= batchLimit) {
                 let start = undefined;
                 if (currentChunk != null) {
                     if (currentChunk.bucket == bucket) {
@@ -379,16 +530,12 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
                     }
                     const yieldChunk = currentChunk;
                     currentChunk = null;
-                    chunkSizeBytes = 0;
-                    yield { chunkData: yieldChunk, targetOp: targetOp };
+                    currentChunkSizeBytes = 0;
+                    yield { chunkData: yieldChunk, targetOp };
                     targetOp = null;
                 }
                 if (start == null) {
-                    const startOpId = bucketMap.get(bucket);
-                    if (startOpId == null) {
-                        throw new Error(`data for unexpected bucket: ${bucket}`);
-                    }
-                    start = internalToExternalOpId(startOpId);
+                    start = internalToExternalOpId(bucketStart);
                 }
                 currentChunk = {
                     bucket,
@@ -398,28 +545,52 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
                     next_after: start
                 };
             }
-            const entry = mapOpEntry(row);
-            if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) {
-                targetOp = row.target_op;
-            }
-            currentChunk.data.push(entry);
-            currentChunk.next_after = entry.op_id;
-            chunkSizeBytes += rawData.byteLength;
+            const rows = extractRowsFromDocument(doc, context, bucketStart, end);
+            currentChunk.data.push(...rows.map(mapOpEntry));
+            currentChunk.next_after = currentChunk.data.at(-1)?.op_id ?? internalToExternalOpId(end);
+            targetOp = maxOpId(targetOp, doc.target_op);
+            currentChunkSizeBytes += doc.size;
         }
         if (currentChunk != null) {
             const yieldChunk = currentChunk;
-            yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups);
-            yield { chunkData: yieldChunk, targetOp: targetOp };
+            // The last chunk may contain more data that was cut in this batch.
+            yieldChunk.has_more = batchHasMore;
+            yield { chunkData: yieldChunk, targetOp };
+        }
+        if (!batchHasMore) {
+            for (const bucket of bucketMap.keys()) {
+                if (!seenBuckets.has(bucket)) {
+                    // We processed everything for this definition group, but this bucket had no data in the batch.
+                    // Yield an empty chunk to indicate that it is complete.
+                    // This prevents re-querying the same bucket in the next batch.
+                    yield {
+                        chunkData: {
+                            bucket,
+                            after: internalToExternalOpId(bucketMap.get(bucket)),
+                            has_more: false,
+                            data: [],
+                            next_after: internalToExternalOpId(end)
+                        },
+                        targetOp
+                    };
+                }
+            }
+        }
+        if (currentChunk != null) {
+            // We yielded data in this group (aside from empty buckets).
+            // Return to the caller to allow them to process it before continuing to the next group.
+            yield { hasMore: batchHasMore || hasLaterDefinitionGroups };
+            break;
         }
-        if (batchHasMore || remainingLimit <= 0) {
-            return;
+        else {
+            // No data in this definition group - continue in the next group.
         }
     }
 }
 export async function getDataBucketChangesV3(ctx, options) {
     const limit = 1000;
     const bucketStateUpdates = await ctx.db
-        .bucketStateV3(ctx.group_id)
+        .bucketState(ctx.replicationStreamId)
         .aggregate([
         {
             $match: {
@@ -449,12 +620,20 @@ export async function getDataBucketChangesV3(ctx, options) {
         updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets)
     };
 }
+/**
+ * Query the parameter entries changed between the two checkpoints, to determine which parameter
+ * lookups need to be re-evaluated.
+ *
+ * This runs at the next checkpoint's snapshot, so it still sees entries that parameter compaction
+ * deleted after that snapshot. Compaction that deleted entries before the snapshot is covered by
+ * the invalidation fence, checked before we get here.
+ */
 export async function getParameterBucketChangesV3(ctx, options) {
     const limit = 1000;
     const indexIds = ctx.mapping.allParameterIndexIds();
     const collections = indexIds.map((indexId) => ({
         indexId,
-        collection: ctx.db.parameterIndexV3(ctx.group_id, indexId)
+        collection: ctx.db.parameterIndex(ctx.replicationStreamId, indexId)
     }));
     if (collections.length == 0) {
         return {
@@ -478,31 +657,44 @@ export async function getParameterBucketChangesV3(ctx, options) {
         }
     ];
     const [firstCollection, ...remainingCollections] = collections;
-    const parameterUpdates = await firstCollection.collection
-        .aggregate([
-        ...pipelineForCollection(firstCollection.indexId),
-        ...remainingCollections.map((collection) => {
-            return {
-                $unionWith: {
-                    coll: collection.collection.collectionName,
-                    pipeline: pipelineForCollection(collection.indexId)
-                }
-            };
-        }),
-        {
-            $limit: limit + 1
-        }
-    ], {
-        batchSize: limit + 2,
-        maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
-    })
-        .toArray();
+    const parameterUpdates = await ctx.db.client.withSession({ snapshot: true }, async (session) => {
+        setSessionSnapshotTime(session, options.nextCheckpoint.snapshotTime);
+        return await firstCollection.collection
+            .aggregate([
+            ...pipelineForCollection(firstCollection.indexId),
+            ...remainingCollections.map((collection) => {
+                return {
+                    $unionWith: {
+                        coll: collection.collection.collectionName,
+                        pipeline: pipelineForCollection(collection.indexId)
+                    }
+                };
+            }),
+            {
+                $limit: limit + 1
+            }
+        ], {
+            session,
+            readConcern: 'snapshot',
+            batchSize: limit + 2,
+            maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
+        })
+            .toArray()
+            .catch((e) => {
+            // Includes the case where the checkpoint snapshot has expired. Degrading to
+            // invalidateParameterBuckets would be safe in itself - it reads nothing - but it gains
+            // nothing: the caller responds to that by re-evaluating the parameter queries at this
+            // same snapshot, which fails too. The checkpoint has to be refetched instead, which is
+            // what the existing sync retry behavior does.
+            throw lib_mongo.mapQueryError(e, 'while querying parameter changes');
+        });
+    });
     const invalidateParameterUpdates = parameterUpdates.length > limit;
     return {
         invalidateParameterBuckets: invalidateParameterUpdates,
         updatedParameterLookups: invalidateParameterUpdates
             ? new Set()
-            : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId))))
+            : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup, p.indexId))))
     };
 }
 //# sourceMappingURL=MongoSyncBucketStorageV3.js.map