@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/dist/storage/implementation/MongoSyncBucketStorage.d.ts

@@ -1,45 +1,66 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { BaseObserver, DO_NOT_LOG, Logger } from '@powersync/lib-services-framework';
-import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, PopulateChecksumCacheOptions, PopulateChecksumCacheResults, ReplicationCheckpoint, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
+import { CheckpointChanges, CompactInitialReplicationOptions, CompactInitialReplicationResults, GetCheckpointChangesOptions, InternalOpId, ReplicationStreamStorageIds, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
 import { HydratedSyncConfig, ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js';
+import { MongoGetCheckpointChangesOptions, MongoSyncBucketStorageCheckpoint } from './common/MongoSyncBucketStorageCheckpoint.js';
 import type { VersionedPowerSyncMongo } from './db.js';
 import { StorageConfig } from './models.js';
 import { MongoBucketBatchOptions } from './MongoBucketBatch.js';
 import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
 import { MongoParameterCompactor } from './MongoParameterCompactor.js';
-import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
+import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
+import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
 export interface MongoSyncBucketStorageOptions {
-    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
+    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+    readPreference?: mongo.ReadPreference;
+    checksumCacheTtlMs?: number;
+    clearBatchThrottleRate?: number;
     storageConfig: StorageConfig;
+    objectStorage?: ObjectStorage;
+    inlineThresholdBytes?: number;
+}
+/**
+ * The stream state read for a checkpoint. All fields must come from the same snapshot.
+ */
+export interface MongoCheckpointState {
+    checkpoint: InternalOpId;
+    lsn: string | null;
+    /** See {@link MongoSyncBucketStorageCheckpoint.parameterChangesInvalidBefore}. */
+    parameterChangesInvalidBefore: InternalOpId;
 }
 interface InternalCheckpointChanges extends CheckpointChanges {
     updatedWriteCheckpoints: Map<string, bigint>;
     invalidateWriteCheckpoints: boolean;
 }
-interface WriterSyncState {
-    lastCheckpointLsn: string | null;
-    resumeFromLsn: string | null;
-    keepaliveOp: InternalOpId | null;
-    syncConfigId?: bson.ObjectId | null;
-}
 export declare abstract class MongoSyncBucketStorage extends BaseObserver<storage.SyncRulesBucketStorageListener> implements storage.SyncRulesBucketStorage {
     #private;
     readonly factory: MongoBucketStorage;
-    readonly group_id: number;
-    protected readonly sync_rules: MongoPersistedSyncRulesContentV1;
-    readonly slot_name: string;
+    readonly replicationStreamId: number;
+    readonly replicationStream: MongoPersistedReplicationStream;
+    readonly replicationStreamName: string;
     readonly db: VersionedPowerSyncMongo;
     [DO_NOT_LOG]: boolean;
     readonly checksums: MongoChecksums;
-    private parsedSyncRulesCache;
+    readonly objectStorage?: ObjectStorage;
+    readonly inlineThresholdBytes: number;
+    /**
+     * Canonical parsed sync config sets, keyed by defaultSchema.
+     *
+     * Entries are never evicted: each parse options value maps to exactly one parsed set for
+     * the lifetime of this storage instance, so parsed source objects and mappings always
+     * stay associated.
+     */
+    private readonly parsedSyncConfigSets;
     private writeCheckpointAPI;
     readonly logger: Logger;
     readonly storageConfig: StorageConfig;
-    constructor(factory: MongoBucketStorage, group_id: number, sync_rules: MongoPersistedSyncRulesContentV1, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
+    readonly readPreference: mongo.ReadPreference | undefined;
+    readonly clearBatchThrottleRate: number;
+    constructor(factory: MongoBucketStorage, replicationStreamId: number, replicationStream: MongoPersistedReplicationStream, replicationStreamName: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
     /**
      * Not for external use - public here for tests only.
      *
@@ -47,47 +68,65 @@ export declare abstract class MongoSyncBucketStorage extends BaseObserver<storag
      */
     abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
     protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
+    protected createWriteCheckpointAPI(options: MongoCheckpointAPIOptions): MongoWriteCheckpointAPI;
     protected abstract createMongoParameterCompactor(checkpoint: InternalOpId, options: storage.CompactOptions): MongoParameterCompactor;
+    /** MongoDB parameter compaction uses a persisted operation-id cursor. */
+    supportsIncrementalParameterCompaction(): boolean;
     get writeCheckpointMode(): storage.WriteCheckpointMode;
-    get mapping(): import("./BucketDefinitionMapping.js").BucketDefinitionMapping;
-    protected get versionContext(): MongoSyncBucketStorageContext;
-    setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void;
-    createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint>;
+    /**
+     * Persisted storage ids of all sync configs in this replication stream. Parse-free.
+     */
+    get storageIds(): ReplicationStreamStorageIds;
+    setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void;
+    createManagedWriteCheckpoints(checkpoints: storage.ManagedWriteCheckpointOptions[]): Promise<storage.CreateManagedWriteCheckpointsResult>;
     lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
-    getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncConfig;
+    getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet;
+    getParsedSyncRules(options: storage.ParseSyncConfigOptions): HydratedSyncConfig;
     getCheckpoint(): Promise<storage.ReplicationCheckpoint>;
-    protected abstract fetchCheckpointState(session: mongo.ClientSession): Promise<{
-        checkpoint: bigint;
-        lsn: string | null;
-    } | null>;
+    protected abstract fetchCheckpointState(session: mongo.ClientSession): Promise<MongoCheckpointState | null>;
     getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null>;
     protected abstract initializeVersionStorage(): Promise<void>;
     private initializeStorage;
-    protected abstract createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch;
-    protected abstract getWriterSyncState(): Promise<WriterSyncState>;
+    /**
+     * Create the version-specific writer. Implementations fetch their own resume state
+     * (e.g. resume LSN, v1 keepalive op) and construct the batch from
+     * {@link writerBatchOptions} plus the version-specific fields.
+     */
+    protected abstract createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
+    /**
+     * The version-independent part of the batch options.
+     */
+    protected writerBatchOptions(options: storage.CreateWriterOptions): Omit<MongoBucketBatchOptions, 'resumeFromLsn'>;
     createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
     startBatch(options: storage.CreateWriterOptions, callback: (batch: storage.BucketStorageBatch) => Promise<void>): Promise<storage.FlushedResult | null>;
     protected abstract getParameterSetsImpl(checkpoint: MongoReplicationCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
     getParameterSets(checkpoint: MongoReplicationCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
-    protected abstract getBucketDataBatchImpl(checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
-    getBucketDataBatch(checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
-    getChecksums(checkpoint: utils.InternalOpId, buckets: storage.BucketChecksumRequest[]): Promise<utils.ChecksumMap>;
+    protected abstract getBucketDataBatchImpl(checkpoint: MongoReplicationCheckpoint, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd>;
+    getBucketDataBatch(checkpoint: storage.ReplicationCheckpoint, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd>;
+    getChecksums(checkpoint: storage.ReplicationCheckpoint, buckets: storage.BucketChecksumRequest[], options?: storage.BucketChecksumOptions): Promise<utils.ChecksumMap>;
     clearChecksumCache(): void;
     protected abstract terminateSyncRuleState(): Promise<void>;
     terminate(options?: storage.TerminateOptions): Promise<void>;
-    protected abstract getStatusImpl(): Promise<storage.SyncRuleStatus>;
-    getStatus(): Promise<storage.SyncRuleStatus>;
+    protected abstract getStatusImpl(): Promise<storage.ReplicationStreamStatus>;
+    getStatus(): Promise<storage.ReplicationStreamStatus>;
     protected abstract clearBucketData(signal?: AbortSignal): Promise<void>;
     protected abstract clearParameterIndexes(signal?: AbortSignal): Promise<void>;
     protected abstract clearSourceRecords(signal?: AbortSignal): Promise<void>;
     protected abstract clearBucketState(signal?: AbortSignal): Promise<void>;
     protected abstract clearSourceTables(signal?: AbortSignal): Promise<void>;
+    protected clearCustomCheckpointRequests(_signal?: AbortSignal): Promise<void>;
     protected abstract clearSyncRuleState(): Promise<void>;
     clear(options?: storage.ClearStorageOptions): Promise<void>;
-    protected clearDeleteMany(label: string, operation: () => Promise<mongo.DeleteResult>, signal?: AbortSignal): Promise<void>;
     reportError(e: any): Promise<void>;
     compact(options?: storage.CompactOptions): Promise<void>;
-    populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults>;
+    abstract compactInitialReplication(options: CompactInitialReplicationOptions): Promise<CompactInitialReplicationResults>;
+    /**
+     * The highest op id persisted for this stream, whether or not covered by a checkpoint.
+     *
+     * Used as the default `maxOpId` for {@link compactInitialReplication}, which runs after
+     * initial replication but before the first checkpoint exists.
+     */
+    protected abstract fetchPersistedOpHead(): Promise<InternalOpId | null>;
     private watchActiveCheckpoint;
     private readonly sharedIter;
     watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable<storage.StorageCheckpointUpdate>;
@@ -95,18 +134,23 @@ export declare abstract class MongoSyncBucketStorage extends BaseObserver<storag
     private checkpointTimeoutStream;
     protected abstract getDataBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
     private getDataBucketChanges;
-    protected abstract getParameterBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
+    protected abstract getParameterBucketChangesImpl(options: MongoGetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
     private getParameterBucketChanges;
     private checkpointChangesCache;
     getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<InternalCheckpointChanges>;
     private getCheckpointChangesInternal;
 }
-declare class MongoReplicationCheckpoint implements ReplicationCheckpoint {
+declare class MongoReplicationCheckpoint implements MongoSyncBucketStorageCheckpoint {
     #private;
     readonly checkpoint: InternalOpId;
     readonly lsn: string | null;
     snapshotTime: mongo.Timestamp;
-    constructor(storage: MongoSyncBucketStorage, checkpoint: InternalOpId, lsn: string | null, snapshotTime: mongo.Timestamp);
+    clusterTime: mongo.ClusterTime;
+    /** Captured in the same snapshot as the checkpoint. */
+    readonly parameterChangesInvalidBefore: InternalOpId;
+    constructor(storage: MongoSyncBucketStorage, checkpoint: InternalOpId, lsn: string | null, snapshotTime: mongo.Timestamp, clusterTime: mongo.ClusterTime, 
+    /** Captured in the same snapshot as the checkpoint. */
+    parameterChangesInvalidBefore: InternalOpId);
     getParameterSets(lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
 }
 export {};

package/dist/storage/implementation/MongoSyncBucketStorage.js

@@ -1,9 +1,10 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { BaseObserver, DO_NOT_LOG, ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
-import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, mergeAsyncIterables, storage } from '@powersync/service-core';
+import { BaseObserver, logger as defaultLogger, DO_NOT_LOG, ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, mergeAsyncIterables, storage, SyncRuleState } from '@powersync/service-core';
 import { LRUCache } from 'lru-cache';
 import * as timers from 'timers/promises';
-import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js';
+import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../../types/types.js';
+import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 /**
  * Only keep checkpoints around for a minute, before fetching a fresh one.
@@ -17,64 +18,102 @@ import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 const CHECKPOINT_TIMEOUT_MS = 60_000;
 export class MongoSyncBucketStorage extends BaseObserver {
     factory;
-    group_id;
-    sync_rules;
-    slot_name;
+    replicationStreamId;
+    replicationStream;
+    replicationStreamName;
     db;
     [DO_NOT_LOG] = true;
     checksums;
-    parsedSyncRulesCache;
+    objectStorage;
+    inlineThresholdBytes;
+    /**
+     * Canonical parsed sync config sets, keyed by defaultSchema.
+     *
+     * Entries are never evicted: each parse options value maps to exactly one parsed set for
+     * the lifetime of this storage instance, so parsed source objects and mappings always
+     * stay associated.
+     */
+    parsedSyncConfigSets = new Map();
     writeCheckpointAPI;
     logger;
     storageConfig;
+    readPreference;
+    clearBatchThrottleRate;
     #storageInitialized = false;
-    constructor(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options) {
+    constructor(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
         super();
         this.factory = factory;
-        this.group_id = group_id;
-        this.sync_rules = sync_rules;
-        this.slot_name = slot_name;
+        this.replicationStreamId = replicationStreamId;
+        this.replicationStream = replicationStream;
+        this.replicationStreamName = replicationStreamName;
         this.storageConfig = options.storageConfig;
+        this.objectStorage = options.objectStorage;
+        // Keep small chunks inline in MongoDB rather than offloading them to S3.
+        // Configurable via object_storage.inline_threshold_bytes.
+        this.inlineThresholdBytes = options.inlineThresholdBytes ?? DEFAULT_INLINE_THRESHOLD_BYTES;
+        this.readPreference = options.readPreference;
+        this.clearBatchThrottleRate = options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
         this.db = factory.db.versioned(this.storageConfig);
         this.checksums = this.createMongoChecksums(options);
-        this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
+        this.writeCheckpointAPI = this.createWriteCheckpointAPI({
             db: this.db,
-            mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
-            sync_rules_id: group_id
+            writeCheckpointMode: { mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED },
+            replicationStreamId
         });
-        this.logger = sync_rules.logger;
+        // Include both replication stream name and sync config version labels in log prefix.
+        // Examples:
+        // No sync config with a label: [stream]
+        // All sync configs have a label: [stream][v5,v6]
+        // Mixed: [stream][?,v6]
+        const hasVersionLabel = replicationStream.syncConfigContent.some((config) => config.version_label != null);
+        const versionLabels = hasVersionLabel
+            ? replicationStream.syncConfigContent.map((config) => config.version_label ?? '?')
+            : [];
+        this.logger =
+            versionLabels.length == 0
+                ? replicationStream.logger
+                : defaultLogger.child({
+                    prefix: `[${replicationStreamName}][${versionLabels.join(',')}] `
+                });
+    }
+    createWriteCheckpointAPI(options) {
+        return new MongoWriteCheckpointAPI(options);
+    }
+    /** MongoDB parameter compaction uses a persisted operation-id cursor. */
+    supportsIncrementalParameterCompaction() {
+        return true;
     }
     get writeCheckpointMode() {
         return this.writeCheckpointAPI.writeCheckpointMode;
     }
-    get mapping() {
-        return this.sync_rules.mapping;
+    /**
+     * Persisted storage ids of all sync configs in this replication stream. Parse-free.
+     */
+    get storageIds() {
+        return this.replicationStream.storageIds;
     }
-    get versionContext() {
-        return {
-            db: this.db,
-            group_id: this.group_id,
-            mapping: this.mapping
-        };
-    }
-    setWriteCheckpointMode(mode) {
-        this.writeCheckpointAPI.setWriteCheckpointMode(mode);
+    setWriteCheckpointMode(config) {
+        this.writeCheckpointAPI.setWriteCheckpointMode(config);
     }
-    createManagedWriteCheckpoint(checkpoint) {
-        return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
+    createManagedWriteCheckpoints(checkpoints) {
+        return this.writeCheckpointAPI.createManagedWriteCheckpoints(checkpoints);
     }
     lastWriteCheckpoint(filters) {
         return this.writeCheckpointAPI.lastWriteCheckpoint({
             ...filters,
-            sync_rules_id: this.group_id
+            sync_rules_id: this.replicationStreamId
         });
     }
-    getParsedSyncRules(options) {
-        const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
-        if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
-            this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncConfig(), options };
+    getParsedSyncConfigSet(options) {
+        let parsed = this.parsedSyncConfigSets.get(options.defaultSchema);
+        if (parsed == null) {
+            parsed = this.replicationStream.parsed(options);
+            this.parsedSyncConfigSets.set(options.defaultSchema, parsed);
         }
-        return this.parsedSyncRulesCache.parsed;
+        return parsed;
+    }
+    getParsedSyncRules(options) {
+        return this.getParsedSyncConfigSet(options).hydratedSyncConfig;
     }
     async getCheckpoint() {
         return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint();
@@ -86,41 +125,47 @@ export class MongoSyncBucketStorage extends BaseObserver {
                 return null;
             }
             const snapshotTime = session.snapshotTime;
+            const clusterTime = session.clusterTime;
             if (snapshotTime == null) {
                 throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
             }
-            return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime);
+            if (clusterTime == null) {
+                throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
+            }
+            return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime, clusterTime, state.parameterChangesInvalidBefore);
         });
     }
     async initializeStorage() {
         if (this.#storageInitialized) {
             return;
         }
-        await this.db.initializeStreamStorage(this.group_id);
+        await this.db.initializeStreamStorage(this.replicationStreamId);
         await this.initializeVersionStorage();
         this.#storageInitialized = true;
     }
-    async createWriter(options) {
-        await this.initializeStorage();
-        const state = await this.getWriterSyncState();
-        const batchOptions = {
+    /**
+     * The version-independent part of the batch options.
+     */
+    writerBatchOptions(options) {
+        return {
             logger: options.logger ?? this.logger,
             db: this.db,
-            syncRules: this.sync_rules.parsed(options).hydratedSyncConfig(),
-            mapping: this.sync_rules.mapping,
-            groupId: this.group_id,
-            slotName: this.slot_name,
-            lastCheckpointLsn: state.lastCheckpointLsn,
-            resumeFromLsn: state.resumeFromLsn,
-            keepaliveOp: state.keepaliveOp,
+            parsedSyncConfig: this.getParsedSyncConfigSet(options),
+            replicationStreamId: this.replicationStreamId,
+            replicationStreamName: this.replicationStreamName,
             storeCurrentData: options.storeCurrentData,
             skipExistingRows: options.skipExistingRows ?? false,
             markRecordUnavailable: options.markRecordUnavailable,
             hooks: options.hooks,
-            syncConfigId: state.syncConfigId,
-            tracer: options.tracer
+            tracer: options.tracer,
+            signal: options.signal,
+            objectStorage: this.objectStorage,
+            inlineThresholdBytes: this.inlineThresholdBytes
         };
-        const writer = this.createWriterImpl(batchOptions);
+    }
+    async createWriter(options) {
+        await this.initializeStorage();
+        const writer = await this.createWriterImpl(options);
         this.iterateListeners((cb) => cb.batchStarted?.(writer));
         return writer;
     }
@@ -136,8 +181,14 @@ export class MongoSyncBucketStorage extends BaseObserver {
     async *getBucketDataBatch(checkpoint, dataBuckets, options) {
         yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options);
     }
-    async getChecksums(checkpoint, buckets) {
-        return this.checksums.getChecksums(checkpoint, buckets);
+    async getChecksums(checkpoint, buckets, options) {
+        const mongoCheckpoint = checkpoint;
+        const snapshotTime = mongoCheckpoint.snapshotTime; // May be undefined in tests
+        return this.checksums.getChecksums(checkpoint.checkpoint, buckets, {
+            snapshotTime,
+            clusterTime: mongoCheckpoint.clusterTime,
+            readPreference: options?.requestHint == 'bulk' ? this.readPreference : undefined
+        });
     }
     clearChecksumCache() {
         this.checksums.clearCache();
@@ -152,6 +203,7 @@ export class MongoSyncBucketStorage extends BaseObserver {
     async getStatus() {
         return this.getStatusImpl();
     }
+    async clearCustomCheckpointRequests(_signal) { }
     async clear(options) {
         const signal = options?.signal;
         if (signal?.aborted) {
@@ -163,22 +215,13 @@ export class MongoSyncBucketStorage extends BaseObserver {
         await this.clearSourceRecords(signal);
         await this.clearBucketState(signal);
         await this.clearSourceTables(signal);
+        await this.clearCustomCheckpointRequests(signal);
         this.#storageInitialized = false;
     }
-    async clearDeleteMany(label, operation, signal) {
-        await retryOnMongoMaxTimeMSExpired(operation, {
-            signal,
-            abortMessage: 'Aborted clearing data',
-            retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5,
-            onRetry: () => {
-                this.logger.info(`Cleared batch of ${label} in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...`);
-            }
-        });
-    }
     async reportError(e) {
         const message = String(e.message ?? 'Replication failure');
         await this.db.sync_rules.updateOne({
-            _id: this.group_id
+            _id: this.replicationStreamId
         }, {
             $set: {
                 last_fatal_error: message,
@@ -189,29 +232,16 @@ export class MongoSyncBucketStorage extends BaseObserver {
     async compact(options) {
         let maxOpId = options?.maxOpId;
         if (maxOpId == null) {
+            // For PROCESSING streams, this will be undefined.
             const checkpoint = await this.getCheckpointInternal();
             maxOpId = checkpoint?.checkpoint ?? undefined;
         }
         await this.createMongoCompactor({ ...options, maxOpId, logger: this.logger }).compact();
-        if (maxOpId != null && options?.compactParameterData) {
-            await this.createMongoParameterCompactor(maxOpId, options).compact();
+        if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.ACTIVE) {
+            // Use the stream-scoped logger, matching bucket compaction above.
+            await this.createMongoParameterCompactor(maxOpId, { ...options, logger: this.logger }).compact();
         }
     }
-    async populatePersistentChecksumCache(options) {
-        this.logger.info(`Populating persistent checksum cache...`);
-        const start = Date.now();
-        const compactor = this.createMongoCompactor({
-            ...options,
-            memoryLimitMB: 0,
-            logger: this.logger
-        });
-        const result = await compactor.populateChecksums({
-            minBucketChanges: options.minBucketChanges ?? 10
-        });
-        const duration = Date.now() - start;
-        this.logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`);
-        return result;
-    }
     async *watchActiveCheckpoint(signal) {
         if (signal.aborted) {
             return;
@@ -239,7 +269,7 @@ export class MongoSyncBucketStorage extends BaseObserver {
         for await (const nextCheckpoint of iter) {
             if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) {
                 writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
-                    sync_rules_id: this.group_id,
+                    sync_rules_id: this.replicationStreamId,
                     user_id: options.user_id,
                     heads: {
                         '1': nextCheckpoint.lsn
@@ -268,7 +298,7 @@ export class MongoSyncBucketStorage extends BaseObserver {
                 let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null;
                 if (updates.invalidateWriteCheckpoints) {
                     updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
-                        sync_rules_id: this.group_id,
+                        sync_rules_id: this.replicationStreamId,
                         user_id: options.user_id,
                         heads: {
                             '1': nextCheckpoint.lsn
@@ -355,11 +385,32 @@ export class MongoSyncBucketStorage extends BaseObserver {
         return this.getDataBucketChangesImpl(options);
     }
     async getParameterBucketChanges(options) {
-        return this.getParameterBucketChangesImpl(options);
+        const nextCheckpoint = requireMongoCheckpoint(options.nextCheckpoint);
+        if (options.lastCheckpoint.checkpoint < nextCheckpoint.parameterChangesInvalidBefore) {
+            // Parameter compaction may have deleted parameter entries in the range we'd have to query
+            // to find the individual changed lookups. Invalidate all parameter buckets instead.
+            //
+            // The fence is committed before the first delete of a compaction pass, and captured in the
+            // same snapshot as the checkpoint, so a checkpoint that could miss a deleted entry always
+            // observes the fence as well.
+            return {
+                invalidateParameterBuckets: true,
+                updatedParameterLookups: new Set()
+            };
+        }
+        // The query below runs at the checkpoint snapshot, which still sees entries deleted by a
+        // compaction pass that started after the snapshot.
+        return this.getParameterBucketChangesImpl({
+            lastCheckpoint: options.lastCheckpoint,
+            nextCheckpoint
+        });
     }
     checkpointChangesCache = new LRUCache({
         max: 50,
         maxSize: 12 * 1024 * 1024,
+        // When we have more fetches than the cache size, complete the fetches instead
+        // of failing with Error('evicted').
+        ignoreFetchAbort: true,
         sizeCalculation: (value) => {
             const paramSize = [...value.updatedParameterLookups].reduce((a, b) => a + b.length, 0);
             const bucketSize = [...value.updatedDataBuckets].reduce((a, b) => a + b.length, 0);
@@ -371,7 +422,10 @@ export class MongoSyncBucketStorage extends BaseObserver {
         }
     });
     async getCheckpointChanges(options) {
-        const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}`;
+        // The invalidation fence is part of the identity: the same checkpoint pair read before and
+        // after a compaction pass produces different results (specific lookups vs. invalidate-all).
+        const fence = requireMongoCheckpoint(options.nextCheckpoint).parameterChangesInvalidBefore;
+        const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}_${fence}`;
         const result = await this.checkpointChangesCache.fetch(key, { context: { options } });
         return result;
     }
@@ -386,25 +440,44 @@ export class MongoSyncBucketStorage extends BaseObserver {
         };
     }
 }
+/**
+ * We don't support any other constructions of ReplicationCheckpoint.
+ */
+function requireMongoCheckpoint(checkpoint) {
+    if (!(checkpoint instanceof MongoReplicationCheckpoint)) {
+        throw new ServiceAssertionError(`Checkpoint changes require a checkpoint from getCheckpointInternal(), got ${checkpoint.constructor.name}`);
+    }
+    return checkpoint;
+}
 class MongoReplicationCheckpoint {
     checkpoint;
     lsn;
     snapshotTime;
+    clusterTime;
+    parameterChangesInvalidBefore;
     #storage;
-    constructor(storage, checkpoint, lsn, snapshotTime) {
+    constructor(storage, checkpoint, lsn, snapshotTime, clusterTime, 
+    /** Captured in the same snapshot as the checkpoint. */
+    parameterChangesInvalidBefore) {
         this.checkpoint = checkpoint;
         this.lsn = lsn;
         this.snapshotTime = snapshotTime;
+        this.clusterTime = clusterTime;
+        this.parameterChangesInvalidBefore = parameterChangesInvalidBefore;
         this.#storage = storage;
     }
     async getParameterSets(lookups, limit) {
         return this.#storage.getParameterSets(this, lookups, limit);
     }
 }
+/**
+ * Used when no checkpoint has been persisted yet. This has no snapshot or invalidation fence, so
+ * it cannot be used for checkpoint change detection - see {@link requireMongoCheckpoint}.
+ */
 class EmptyReplicationCheckpoint {
     checkpoint = 0n;
     lsn = null;
-    async getParameterSets(_lookups) {
+    async getParameterSets(_lookups, _limit) {
         return [];
     }
 }