@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/src/storage/implementation/MongoPersistedSyncConfigContent.ts

@@ -0,0 +1,152 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import {
+  ReplicationStreamDocumentV3,
+  SyncConfigDefinition,
+  SyncRuleConfigStateV3,
+  SyncRuleDocumentV1
+} from '../storage-index.js';
+import { PowerSyncMongo } from './db.js';
+import { getMongoStorageConfig } from './models.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+
+export abstract class MongoPersistedSyncConfigContentBase extends storage.PersistedSyncConfigContent {
+  public readonly mapping: SingleSyncConfigBucketDefinitionMapping;
+  public readonly syncConfigObjectId: bson.ObjectId | null;
+
+  protected constructor(
+    protected readonly db: PowerSyncMongo,
+    options: Omit<storage.PersistedSyncConfigContentData, 'syncConfigId'> & {
+      mapping: SingleSyncConfigBucketDefinitionMapping;
+      syncConfigId: bson.ObjectId | null;
+    }
+  ) {
+    const { mapping, syncConfigId, ...base } = options;
+    super({
+      ...base,
+      syncConfigId: syncConfigId?.toHexString() ?? null
+    });
+    this.mapping = mapping;
+    this.syncConfigObjectId = syncConfigId;
+  }
+
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
+  }
+
+  parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
+    const parsed = super.parsed(options);
+    const storageConfig = this.getStorageConfig();
+    const [syncConfig] = parsed.syncConfigs;
+    if (syncConfig == null) {
+      throw new ServiceAssertionError(`Expected one parsed sync config`);
+    }
+
+    if (storageConfig.incrementalReprocessing) {
+      const syncConfigId = this.syncConfigId;
+      if (syncConfigId == null) {
+        throw new ServiceAssertionError(`syncConfigId is required for v3 sync config content`);
+      }
+
+      return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
+        { syncConfigId, syncConfig, mapping: this.mapping }
+      ]);
+    }
+
+    return new MongoParsedSyncConfigSet(parsed.replicationStreamId, storageConfig, parsed.replicationStreamName, [
+      { syncConfig, mapping: null }
+    ]);
+  }
+}
+export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase {
+  constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1) {
+    super(db, {
+      replicationStreamId: doc._id,
+      sync_rules_content: doc.content,
+      compiled_plan: doc.serialized_plan ?? null,
+      // Handle legacy values
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
+      mapping: new SingleSyncConfigBucketDefinitionMapping(),
+      syncConfigId: null
+    });
+  }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    const doc = await this.db.sync_rules.findOne<SyncRuleDocumentV1>({ _id: this.replicationStreamId });
+    if (doc == null) {
+      return null;
+    }
+
+    return syncConfigStatusFromV1(doc);
+  }
+}
+export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase {
+  declare public readonly syncConfigObjectId: bson.ObjectId;
+
+  constructor(db: PowerSyncMongo, doc: ReplicationStreamDocumentV3, config: SyncConfigDefinition) {
+    const state = doc.sync_configs.find((c) => c._id.equals(config._id));
+    if (state == null) {
+      throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`);
+    }
+    super(db, {
+      replicationStreamId: doc._id,
+      sync_rules_content: config.content,
+      compiled_plan: config.serialized_plan ?? null,
+
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      storageVersion: doc.storage_version,
+      mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
+      syncConfigId: config._id
+    });
+  }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    const doc = await this.db.sync_rules.findOne<ReplicationStreamDocumentV3>({
+      _id: this.replicationStreamId,
+      'sync_configs._id': this.syncConfigObjectId
+    });
+    if (doc == null) {
+      return null;
+    }
+
+    const state = doc.sync_configs.find((c) => c._id.equals(this.syncConfigObjectId));
+    if (state == null) {
+      return null;
+    }
+
+    return syncConfigStatusFromV3(doc, state);
+  }
+}
+
+function syncConfigStatusFromV1(doc: SyncRuleDocumentV1): storage.PersistedSyncConfigStatus {
+  return {
+    id: String(doc._id),
+    replicationStreamId: doc._id,
+    state: doc.state,
+    snapshot_done: doc.snapshot_done,
+    last_checkpoint_lsn: doc.last_checkpoint_lsn,
+    last_fatal_error: doc.last_fatal_error,
+    last_fatal_error_ts: doc.last_fatal_error_ts,
+    last_keepalive_ts: doc.last_keepalive_ts,
+    last_checkpoint_ts: doc.last_checkpoint_ts
+  };
+}
+
+function syncConfigStatusFromV3(
+  doc: ReplicationStreamDocumentV3,
+  state: SyncRuleConfigStateV3
+): storage.PersistedSyncConfigStatus {
+  return {
+    id: state._id.toHexString(),
+    replicationStreamId: doc._id,
+    state: state.state,
+    snapshot_done: state.snapshot_done,
+    last_checkpoint_lsn: state.last_checkpoint_lsn,
+    last_fatal_error: doc.last_fatal_error,
+    last_fatal_error_ts: doc.last_fatal_error_ts,
+    last_keepalive_ts: doc.last_keepalive_ts,
+    last_checkpoint_ts: doc.last_checkpoint_ts
+  };
+}

package/src/storage/implementation/MongoStorageProvider.ts

@@ -39,8 +39,11 @@ export class MongoStorageProvider implements storage.StorageProvider {
 
     const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
     const syncStorageFactory = new MongoBucketStorage(database, {
-      // TODO currently need the entire resolved config due to this
-      slot_name_prefix: resolvedConfig.slot_name_prefix
+      replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
+      // Right now, only MongoDB source databases supports incremental reprocessing.
+      // Remove this filter when we support it for other source databases.
+      // This assumes a single source connection - revisit if we ever support multiple connections.
+      supportsMultipleSyncConfigs: resolvedConfig.connections?.[0]?.type == lib_mongo.MONGO_CONNECTION_TYPE
     });
 
     // Storage factory for reports

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -13,33 +13,33 @@ import {
   CheckpointChanges,
   GetCheckpointChangesOptions,
   InternalOpId,
-  maxLsn,
   mergeAsyncIterables,
   PopulateChecksumCacheOptions,
   PopulateChecksumCacheResults,
   ReplicationCheckpoint,
+  ReplicationStreamStorageIds,
   storage,
   utils,
   WatchWriteCheckpointOptions
 } from '@powersync/service-core';
-import { HydratedSyncRules, ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
+import { HydratedSyncConfig, ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { LRUCache } from 'lru-cache';
 import * as timers from 'timers/promises';
 import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
-import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js';
 import type { VersionedPowerSyncMongo } from './db.js';
-import { CommonSourceTableDocument, StorageConfig } from './models.js';
+import { StorageConfig } from './models.js';
 import { MongoBucketBatchOptions } from './MongoBucketBatch.js';
 import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
 import { MongoParameterCompactor } from './MongoParameterCompactor.js';
-import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
 
 export interface MongoSyncBucketStorageOptions {
-  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
   storageConfig: StorageConfig;
 }
 
@@ -64,32 +64,42 @@ export abstract class MongoSyncBucketStorage
   implements storage.SyncRulesBucketStorage
 {
   readonly db: VersionedPowerSyncMongo;
+
   [DO_NOT_LOG] = true;
 
   readonly checksums: MongoChecksums;
 
-  private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined;
+  /**
+   * Canonical parsed sync config sets, keyed by defaultSchema.
+   *
+   * Entries are never evicted: each parse options value maps to exactly one parsed set for
+   * the lifetime of this storage instance, so parsed source objects and mappings always
+   * stay associated.
+   */
+  private readonly parsedSyncConfigSets = new Map<string, MongoParsedSyncConfigSet>();
   private writeCheckpointAPI: MongoWriteCheckpointAPI;
   public readonly logger: Logger;
+  public readonly storageConfig: StorageConfig;
   #storageInitialized = false;
 
   constructor(
     public readonly factory: MongoBucketStorage,
-    public readonly group_id: number,
-    protected readonly sync_rules: MongoPersistedSyncRulesContent,
-    public readonly slot_name: string,
+    public readonly replicationStreamId: number,
+    public readonly replicationStream: MongoPersistedReplicationStream,
+    public readonly replicationStreamName: string,
     writeCheckpointMode: storage.WriteCheckpointMode | undefined,
     options: MongoSyncBucketStorageOptions
   ) {
     super();
-    this.db = factory.db.versioned(sync_rules.getStorageConfig());
+    this.storageConfig = options.storageConfig;
+    this.db = factory.db.versioned(this.storageConfig);
     this.checksums = this.createMongoChecksums(options);
     this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
       db: this.db,
       mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
-      sync_rules_id: group_id
+      sync_rules_id: replicationStreamId
     });
-    this.logger = sync_rules.logger;
+    this.logger = replicationStream.logger;
   }
 
   /**
@@ -109,16 +119,11 @@ export abstract class MongoSyncBucketStorage
     return this.writeCheckpointAPI.writeCheckpointMode;
   }
 
-  get mapping() {
-    return this.sync_rules.mapping;
-  }
-
-  protected get versionContext(): MongoSyncBucketStorageContext {
-    return {
-      db: this.db,
-      group_id: this.group_id,
-      mapping: this.mapping
-    };
+  /**
+   * Persisted storage ids of all sync configs in this replication stream. Parse-free.
+   */
+  get storageIds(): ReplicationStreamStorageIds {
+    return this.replicationStream.storageIds;
   }
 
   setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
@@ -132,33 +137,35 @@ export abstract class MongoSyncBucketStorage
   lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
     return this.writeCheckpointAPI.lastWriteCheckpoint({
       ...filters,
-      sync_rules_id: this.group_id
+      sync_rules_id: this.replicationStreamId
     });
   }
 
-  getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules {
-    const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
-    if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
-      this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options };
+  getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
+    let parsed = this.parsedSyncConfigSets.get(options.defaultSchema);
+    if (parsed == null) {
+      parsed = this.replicationStream.parsed(options);
+      this.parsedSyncConfigSets.set(options.defaultSchema, parsed);
     }
+    return parsed;
+  }
 
-    return this.parsedSyncRulesCache!.parsed;
+  getParsedSyncRules(options: storage.ParseSyncConfigOptions): HydratedSyncConfig {
+    return this.getParsedSyncConfigSet(options).hydratedSyncConfig;
   }
 
   async getCheckpoint(): Promise<storage.ReplicationCheckpoint> {
     return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint();
   }
 
+  protected abstract fetchCheckpointState(
+    session: mongo.ClientSession
+  ): Promise<{ checkpoint: bigint; lsn: string | null } | null>;
+
   async getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null> {
     return await this.db.client.withSession({ snapshot: true }, async (session) => {
-      const doc = await this.db.sync_rules.findOne(
-        { _id: this.group_id },
-        {
-          session,
-          projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
-        }
-      );
-      if (!doc?.snapshot_done || !['ACTIVE', 'ERRORED'].includes(doc.state)) {
+      const state = await this.fetchCheckpointState(session);
+      if (state == null) {
         return null;
       }
 
@@ -166,12 +173,7 @@ export abstract class MongoSyncBucketStorage
       if (snapshotTime == null) {
         throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
       }
-      return new MongoReplicationCheckpoint(
-        this,
-        doc.last_checkpoint ?? 0n,
-        doc.last_checkpoint_lsn ?? null,
-        snapshotTime
-      );
+      return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime);
     });
   }
 
@@ -182,40 +184,40 @@ export abstract class MongoSyncBucketStorage
       return;
     }
 
-    await this.db.initializeStreamStorage(this.group_id);
+    await this.db.initializeStreamStorage(this.replicationStreamId);
     await this.initializeVersionStorage();
     this.#storageInitialized = true;
   }
 
-  protected abstract createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch;
-
-  async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
-    await this.initializeStorage();
-
-    const doc = await this.db.sync_rules.findOne(
-      {
-        _id: this.group_id
-      },
-      { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1, keepalive_op: 1, snapshot_lsn: 1 } }
-    );
-    const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null;
+  /**
+   * Create the version-specific writer. Implementations fetch their own resume state
+   * (e.g. resume LSN, v1 keepalive op) and construct the batch from
+   * {@link writerBatchOptions} plus the version-specific fields.
+   */
+  protected abstract createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
 
-    const batchOptions: MongoBucketBatchOptions = {
+  /**
+   * The version-independent part of the batch options.
+   */
+  protected writerBatchOptions(options: storage.CreateWriterOptions): Omit<MongoBucketBatchOptions, 'resumeFromLsn'> {
+    return {
       logger: options.logger ?? this.logger,
       db: this.db,
-      syncRules: this.sync_rules.parsed(options).hydratedSyncRules(),
-      mapping: this.sync_rules.mapping,
-      groupId: this.group_id,
-      slotName: this.slot_name,
-      lastCheckpointLsn: checkpoint_lsn,
-      resumeFromLsn: maxLsn(checkpoint_lsn, doc?.snapshot_lsn),
-      keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
+      parsedSyncConfig: this.getParsedSyncConfigSet(options),
+      replicationStreamId: this.replicationStreamId,
+      replicationStreamName: this.replicationStreamName,
       storeCurrentData: options.storeCurrentData,
       skipExistingRows: options.skipExistingRows ?? false,
       markRecordUnavailable: options.markRecordUnavailable,
+      hooks: options.hooks,
       tracer: options.tracer
     };
-    const writer = this.createWriterImpl(batchOptions);
+  }
+
+  async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
+    await this.initializeStorage();
+
+    const writer = await this.createWriterImpl(options);
     this.iterateListeners((cb) => cb.batchStarted?.(writer));
     return writer;
   }
@@ -230,134 +232,6 @@ export abstract class MongoSyncBucketStorage
     return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null;
   }
 
-  protected abstract sourceTableBaseId(): Partial<CommonSourceTableDocument>;
-
-  protected abstract augmentCreatedSourceTableDocument(
-    createDoc: CommonSourceTableDocument,
-    options: storage.ResolveTableOptions,
-    candidateSourceTable: storage.SourceTable
-  ): void;
-
-  protected abstract initializeResolvedSourceRecords(sourceTableId: bson.ObjectId): Promise<void>;
-
-  async resolveTable(options: storage.ResolveTableOptions): Promise<storage.ResolveTableResult> {
-    const { group_id, connection_id, connection_tag, entity_descriptor } = options;
-
-    const { schema, name, objectId, replicaIdColumns } = entity_descriptor;
-
-    const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
-      name: column.name,
-      type: column.type,
-      type_oid: column.typeId
-    }));
-    let result: storage.ResolveTableResult | null = null;
-    let initializeSourceRecordsFor: bson.ObjectId | null = null;
-
-    const baseId = this.sourceTableBaseId();
-    await this.db.client.withSession(async (session) => {
-      const col = this.db.commonSourceTables(group_id);
-      let filter: Partial<CommonSourceTableDocument> = {
-        ...baseId,
-        connection_id: connection_id,
-        schema_name: schema,
-        table_name: name,
-        replica_id_columns2: normalizedReplicaIdColumns
-      };
-
-      if (objectId != null) {
-        filter.relation_id = objectId;
-      }
-      let doc = await col.findOne(filter, { session });
-      if (doc == null) {
-        const candidateSourceTable = new storage.SourceTable({
-          id: new bson.ObjectId(),
-          connectionTag: connection_tag,
-          objectId: objectId,
-          schema: schema,
-          name: name,
-          replicaIdColumns: replicaIdColumns,
-          snapshotComplete: false
-        });
-        const createDoc: CommonSourceTableDocument = {
-          _id: candidateSourceTable.id as bson.ObjectId,
-          ...(baseId as any),
-          connection_id: connection_id,
-          relation_id: objectId,
-          schema_name: schema,
-          table_name: name,
-          replica_id_columns: null,
-          replica_id_columns2: normalizedReplicaIdColumns,
-          snapshot_done: false,
-          snapshot_status: undefined
-        };
-        this.augmentCreatedSourceTableDocument(createDoc, options, candidateSourceTable);
-        doc = createDoc;
-
-        await col.insertOne(doc, { session });
-        initializeSourceRecordsFor = doc._id;
-      }
-      const sourceTable = new storage.SourceTable({
-        id: doc._id,
-        connectionTag: connection_tag,
-        objectId: objectId,
-        schema: schema,
-        name: name,
-        replicaIdColumns: replicaIdColumns,
-        snapshotComplete: doc.snapshot_done ?? true
-      });
-      sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable);
-      sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable);
-      sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable);
-      sourceTable.snapshotStatus =
-        doc.snapshot_status == null
-          ? undefined
-          : {
-              lastKey: doc.snapshot_status.last_key?.buffer ?? null,
-              totalEstimatedCount: doc.snapshot_status.total_estimated_count,
-              replicatedCount: doc.snapshot_status.replicated_count
-            };
-
-      let dropTables: storage.SourceTable[] = [];
-      let truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
-      if (objectId != null) {
-        truncateFilter.push({ relation_id: objectId });
-      }
-      const truncate = await col
-        .find(
-          {
-            ...baseId,
-            connection_id: connection_id,
-            _id: { $ne: doc._id },
-            $or: truncateFilter
-          },
-          { session }
-        )
-        .toArray();
-      dropTables = truncate.map(
-        (doc) =>
-          new storage.SourceTable({
-            id: doc._id,
-            connectionTag: connection_tag,
-            objectId: doc.relation_id,
-            schema: doc.schema_name,
-            name: doc.table_name,
-            replicaIdColumns:
-              doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [],
-            snapshotComplete: doc.snapshot_done ?? true
-          })
-      );
-
-      result = {
-        table: sourceTable,
-        dropTables: dropTables
-      };
-    });
-    if (initializeSourceRecordsFor != null) {
-      await this.initializeResolvedSourceRecords(initializeSourceRecordsFor);
-    }
-    return result!;
-  }
-
   protected abstract getParameterSetsImpl(
     checkpoint: MongoReplicationCheckpoint,
     lookups: ScopedParameterLookup[],
@@ -397,49 +271,20 @@ export abstract class MongoSyncBucketStorage
     this.checksums.clearCache();
   }
 
+  protected abstract terminateSyncRuleState(): Promise<void>;
+
   async terminate(options?: storage.TerminateOptions) {
     if (!options || options?.clearStorage) {
       await this.clear(options);
     }
-    await this.db.sync_rules.updateOne(
-      {
-        _id: this.group_id
-      },
-      {
-        $set: {
-          state: storage.SyncRuleState.TERMINATED,
-          persisted_lsn: null,
-          snapshot_done: false
-        }
-      }
-    );
+    await this.terminateSyncRuleState();
     await this.db.notifyCheckpoint();
   }
 
-  async getStatus(): Promise<storage.SyncRuleStatus> {
-    const doc = await this.db.sync_rules.findOne(
-      {
-        _id: this.group_id
-      },
-      {
-        projection: {
-          snapshot_done: 1,
-          last_checkpoint_lsn: 1,
-          state: 1,
-          snapshot_lsn: 1
-        }
-      }
-    );
-    if (doc == null) {
-      throw new ServiceAssertionError('Cannot find replication stream status');
-    }
+  protected abstract getStatusImpl(): Promise<storage.ReplicationStreamStatus>;
 
-    return {
-      snapshot_done: doc.snapshot_done,
-      snapshot_lsn: doc.snapshot_lsn ?? null,
-      active: doc.state == 'ACTIVE',
-      checkpoint_lsn: doc.last_checkpoint_lsn
-    };
+  async getStatus(): Promise<storage.ReplicationStreamStatus> {
+    return this.getStatusImpl();
   }
 
   protected abstract clearBucketData(signal?: AbortSignal): Promise<void>;
@@ -451,6 +296,7 @@ export abstract class MongoSyncBucketStorage
   protected abstract clearBucketState(signal?: AbortSignal): Promise<void>;
 
   protected abstract clearSourceTables(signal?: AbortSignal): Promise<void>;
+  protected abstract clearSyncRuleState(): Promise<void>;
 
   async clear(options?: storage.ClearStorageOptions): Promise<void> {
     const signal = options?.signal;
@@ -459,24 +305,7 @@ export abstract class MongoSyncBucketStorage
       throw new ReplicationAbortedError('Aborted clearing data', signal.reason);
     }
 
-    await this.db.sync_rules.updateOne(
-      {
-        _id: this.group_id
-      },
-      {
-        $set: {
-          snapshot_done: false,
-          persisted_lsn: null,
-          last_checkpoint_lsn: null,
-          last_checkpoint: null,
-          no_checkpoint_before: null
-        },
-        $unset: {
-          snapshot_lsn: 1
-        }
-      },
-      { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
-    );
+    await this.clearSyncRuleState();
 
     await this.clearBucketData(signal);
     await this.clearParameterIndexes(signal);
@@ -508,7 +337,7 @@ export abstract class MongoSyncBucketStorage
     const message = String(e.message ?? 'Replication failure');
     await this.db.sync_rules.updateOne(
       {
-        _id: this.group_id
+        _id: this.replicationStreamId
       },
       {
         $set: {
@@ -532,11 +361,21 @@ export abstract class MongoSyncBucketStorage
     }
   }
 
+  /**
+   * The highest op id persisted for this stream, whether or not covered by a checkpoint.
+   *
+   * Used as the default `maxOpId` for {@link populatePersistentChecksumCache}, which runs after
+   * initial replication but before the first checkpoint exists.
+   */
+  protected abstract fetchPersistedOpHead(): Promise<InternalOpId | null>;
+
   async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
     this.logger.info(`Populating persistent checksum cache...`);
     const start = Date.now();
+    const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
     const compactor = this.createMongoCompactor({
       ...options,
+      maxOpId,
       memoryLimitMB: 0,
       logger: this.logger
     });
@@ -588,7 +427,7 @@ export abstract class MongoSyncBucketStorage
     for await (const nextCheckpoint of iter) {
       if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) {
         writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
-          sync_rules_id: this.group_id,
+          sync_rules_id: this.replicationStreamId,
           user_id: options.user_id,
           heads: {
             '1': nextCheckpoint.lsn
@@ -621,7 +460,7 @@ export abstract class MongoSyncBucketStorage
         let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null;
         if (updates.invalidateWriteCheckpoints) {
           updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
-            sync_rules_id: this.group_id,
+            sync_rules_id: this.replicationStreamId,
             user_id: options.user_id,
             heads: {
               '1': nextCheckpoint.lsn!
@@ -792,7 +631,7 @@ class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
   readonly checkpoint: InternalOpId = 0n;
   readonly lsn: string | null = null;
 
-  async getParameterSets(_lookups: ScopedParameterLookup[]): Promise<ParameterLookupRows[]> {
+  async getParameterSets(_lookups: ScopedParameterLookup[], _limit: number): Promise<ParameterLookupRows[]> {
     return [];
   }
 }