@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/src/storage/implementation/MongoChecksums.ts

@@ -7,18 +7,16 @@ import {
   ChecksumMap,
   FetchPartialBucketChecksum,
   InternalOpId,
-  isPartialChecksum,
   PartialChecksum,
   PartialChecksumMap,
   PartialOrFullChecksum
 } from '@powersync/service-core';
 import type { VersionedPowerSyncMongo } from './db.js';
 
-import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js';
-import { BucketDataDocumentBase, StorageConfig } from './models.js';
+import { BucketDefinitionId } from '@powersync/service-sync-rules';
+import { StorageConfig } from './models.js';
 
-export interface FetchPartialBucketChecksumV3 {
+export interface FetchPartialBucketChecksumByDefinition {
   bucket: string;
   definitionId: BucketDefinitionId;
   start?: InternalOpId;
@@ -46,15 +44,14 @@ export interface MongoChecksumOptions {
   operationBatchLimit?: number;
 
   storageConfig: StorageConfig;
-  mapping?: BucketDefinitionMapping;
 }
 
 const DEFAULT_BUCKET_BATCH_LIMIT = 200;
-const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
+export const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
 
 export abstract class MongoChecksums {
   private _cache: ChecksumCache | undefined;
-  private readonly storageConfig: StorageConfig;
+  protected readonly storageConfig: StorageConfig;
 
   constructor(
     protected readonly db: VersionedPowerSyncMongo,
@@ -177,138 +174,10 @@ export abstract class MongoChecksums {
   protected abstract fetchPreStates(
     batch: FetchPartialBucketChecksum[]
   ): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>>;
-
-  protected async computePartialChecksumsForCollection<
-    TRequest extends FetchPartialBucketChecksumByBucket,
-    TBucketDataDocument extends BucketDataDocumentBase
-  >(
-    batch: TRequest[],
-    collection: lib_mongo.mongo.Collection<TBucketDataDocument>,
-    createFilter: (request: TRequest) => any
-  ): Promise<PartialChecksumMap> {
-    const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT;
-
-    // Map requests by bucket. We adjust this as we get partial results.
-    let requests = new Map<string, TRequest>();
-    for (let request of batch) {
-      requests.set(request.bucket, request);
-    }
-
-    const partialChecksums = new Map<string, PartialOrFullChecksum>();
-
-    while (requests.size > 0) {
-      const filters = Array.from(requests.values(), createFilter);
-
-      // Historically, checksum may be stored as 'int' or 'double'.
-      // More recently, this should be a 'long'.
-      // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-      const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' };
-
-      // Aggregate over a max of `batchLimit` operations at a time.
-      // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
-      // Then we'll do three batches:
-      // 1. Query: A[1-end], B[1-end], C[1-end]
-      //    Returns: A[1-10], B[1-2]
-      // 2. Query: B[3-end], C[1-end]
-      //    Returns: B[3-10], C[1-4]
-      // 3. Query: C[5-end]
-      //    Returns: C[5-10]
-      const aggregate = await collection
-        .aggregate(
-          [
-            {
-              $match: {
-                $or: filters
-              }
-            },
-            // sort and limit _before_ grouping
-            { $sort: { _id: 1 } },
-            { $limit: batchLimit },
-            {
-              $group: {
-                _id: '$_id.b',
-                checksum_total: { $sum: checksumLong },
-                count: { $sum: 1 },
-                has_clear_op: {
-                  $max: {
-                    $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
-                  }
-                },
-                last_op: { $max: '$_id.o' }
-              }
-            },
-            // Sort the aggregated results (100 max, so should be fast).
-            // This is important to identify which buckets we have partial data for.
-            { $sort: { _id: 1 } }
-          ],
-          { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS }
-        )
-        .toArray()
-        .catch((e) => {
-          throw lib_mongo.mapQueryError(e, 'while reading checksums');
-        });
-
-      let batchCount = 0;
-      let limitReached = false;
-      for (let doc of aggregate) {
-        const bucket = doc._id;
-        const checksum = checksumFromAggregate(doc);
-
-        const existing = partialChecksums.get(bucket);
-        if (existing != null) {
-          partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum));
-        } else {
-          partialChecksums.set(bucket, checksum);
-        }
-
-        batchCount += doc.count;
-        if (batchCount == batchLimit) {
-          // Limit reached. Request more in the next batch.
-          // Note that this only affects the _last_ bucket in a batch.
-          limitReached = true;
-          const req = requests.get(bucket);
-          requests.set(bucket, {
-            ...req!,
-            start: doc.last_op
-          });
-        } else {
-          // All done for this bucket
-          requests.delete(bucket);
-        }
-      }
-      if (!limitReached) {
-        break;
-      }
-    }
-
-    return new Map<string, PartialOrFullChecksum>(
-      batch.map((request) => {
-        const bucket = request.bucket;
-        // Could be null if we got no data
-        let partialChecksum = partialChecksums.get(bucket);
-        if (partialChecksum == null) {
-          partialChecksum = {
-            bucket,
-            partialCount: 0,
-            partialChecksum: 0
-          };
-        }
-        if (request.start == null && isPartialChecksum(partialChecksum)) {
-          partialChecksum = {
-            bucket,
-            count: partialChecksum.partialCount,
-            checksum: partialChecksum.partialChecksum
-          };
-        }
-
-        return [bucket, partialChecksum];
-      })
-    );
-  }
 }
 
 export function emptyChecksumForRequest(
-  request: Pick<FetchPartialBucketChecksum | FetchPartialBucketChecksumV3, 'bucket' | 'start'>
+  request: Pick<FetchPartialBucketChecksum | FetchPartialBucketChecksumByDefinition, 'bucket' | 'start'>
 ): PartialOrFullChecksum {
   if (request.start == null) {
     return { bucket: request.bucket, count: 0, checksum: 0 };
@@ -319,7 +188,7 @@ export function emptyChecksumForRequest(
 /**
  * Convert output of the $group stage into a checksum.
  */
-function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum {
+export function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum {
   const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff;
   const bucket = doc._id;
 

package/src/storage/implementation/MongoCompactor.ts

@@ -13,8 +13,8 @@ import {
   storage,
   utils
 } from '@powersync/service-core';
+import { BucketDefinitionId } from '@powersync/service-sync-rules';
 
-import { BucketDefinitionId } from './BucketDefinitionMapping.js';
 import { BucketDataDoc, BucketKey } from './common/BucketDataDoc.js';
 import { BucketDataDocumentGeneric, SingleBucketStore } from './common/SingleBucketStore.js';
 import type { VersionedPowerSyncMongo } from './db.js';
@@ -63,6 +63,7 @@ export interface MongoCompactOptions extends storage.CompactOptions {}
 const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
+const DEFAULT_MOVE_BATCH_BYTE_LIMIT = 16 * 1024 * 1024;
 const DEFAULT_MIN_BUCKET_CHANGES = 10;
 const DEFAULT_MIN_CHANGE_RATIO = 0.1;
 const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
@@ -83,6 +84,7 @@ export abstract class MongoCompactor {
   protected readonly idLimitBytes: number;
   protected readonly moveBatchLimit: number;
   protected readonly moveBatchQueryLimit: number;
+  protected readonly moveBatchByteLimit: number;
   protected readonly clearBatchLimit: number;
   protected readonly minBucketChanges: number;
   protected readonly minChangeRatio: number;
@@ -98,11 +100,15 @@ export abstract class MongoCompactor {
     protected readonly db: VersionedPowerSyncMongo,
     options: MongoCompactOptions
   ) {
-    this.group_id = storage.group_id;
+    this.group_id = storage.replicationStreamId;
     this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
     this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
     this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
+    this.moveBatchByteLimit = options.moveBatchByteLimit ?? DEFAULT_MOVE_BATCH_BYTE_LIMIT;
     this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
+    if (this.clearBatchLimit < 2) {
+      throw new ReplicationAssertionError('clearBatchLimit must be >= 2');
+    }
     this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
     this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
     this.maxOpId = options.maxOpId ?? 0n;
@@ -133,6 +139,7 @@ export abstract class MongoCompactor {
    */
   async populateChecksums(options: { minBucketChanges: number }): Promise<PopulateChecksumCacheResults> {
     let count = 0;
+    // Paginate through dirty buckets in batches until no more buckets meet the criteria.
     while (true) {
       this.signal?.throwIfAborted();
       const buckets = await this.dirtyBucketBatchForChecksums(options);
@@ -172,6 +179,7 @@ export abstract class MongoCompactor {
     },
     getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null
   ): AsyncGenerator<DirtyBucket[]> {
+    // Paginate through the bucket state collection using cursor-based scanning.
     while (true) {
       // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline
       // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria.
@@ -304,6 +312,7 @@ export abstract class MongoCompactor {
    */
   protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) {
     let retryCount = 0;
+    // Retry with exponential backoff up to 3 times on MongoDB errors.
     while (true) {
       try {
         await this.compactSingleBucket(bucket, definitionId);
@@ -343,6 +352,7 @@ export abstract class MongoCompactor {
     // Upper bound is adjusted for each batch.
     let upperBound = bucketContext.maxId;
 
+    // Paginate through bucket data in batches to avoid cursor timeouts.
     while (true) {
       this.signal?.throwIfAborted();
 
@@ -353,11 +363,7 @@ export abstract class MongoCompactor {
             _id: {
               $gte: lowerBound,
               $lt: upperBound
-            },
-            // Workaround for a clustered collection bug where the $lt operator may include upperBound.
-            // Technically only needed for storage V3.
-            // https://jira.mongodb.org/browse/SERVER-121822
-            '_id.o': { $lt: upperBound.o }
+            }
           }
         },
         { $sort: { _id: -1 } },
@@ -488,13 +494,13 @@ export abstract class MongoCompactor {
     await this.flush(bucketContext);
   }
 
-  protected updateBucketChecksums(state: CurrentBucketState) {
+  protected collectBucketStateUpdates(state: CurrentBucketState): mongo.AnyBulkWriteOperation<BucketStateDocumentBase> {
     if (state.opCount < 0) {
       throw new ServiceAssertionError(
         `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`
       );
     }
-    this.bucketStateUpdates.push({
+    return {
       updateOne: {
         filter: this.bucketStateFilter(state.bucket, state.definitionId),
         update: {
@@ -517,7 +523,11 @@ export abstract class MongoCompactor {
         // We don't create new ones here, to avoid issues with the unique index on bucket_updates.
         upsert: false
       }
-    });
+    };
+  }
+
+  protected updateBucketChecksums(state: CurrentBucketState) {
+    this.bucketStateUpdates.push(this.collectBucketStateUpdates(state));
   }
 
   protected async flush(col: SingleBucketStore) {

package/src/storage/implementation/MongoHydrationState.ts

@@ -0,0 +1,61 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { SyncConfigWithRequiredMapping } from '@powersync/service-core';
+import {
+  BucketDataScope,
+  BucketDataSource,
+  HydrationState,
+  ParameterIndexLookupCreator,
+  ParameterLookupScope
+} from '@powersync/service-sync-rules';
+
+export class MongoHydrationState implements HydrationState {
+  private bucketDataSourceSyncConfig = new WeakMap<BucketDataSource, SyncConfigWithRequiredMapping>();
+  private parameterIndexLookupSyncConfig = new WeakMap<ParameterIndexLookupCreator, SyncConfigWithRequiredMapping>();
+
+  constructor(
+    readonly syncConfigs: SyncConfigWithRequiredMapping[],
+    private readonly replicationStreamId: number
+  ) {
+    for (let syncConfig of syncConfigs) {
+      for (let source of syncConfig.syncConfig.config.bucketDataSources) {
+        this.bucketDataSourceSyncConfig.set(source, syncConfig);
+      }
+      for (let source of syncConfig.syncConfig.config.bucketParameterLookupSources) {
+        this.parameterIndexLookupSyncConfig.set(source, syncConfig);
+      }
+    }
+  }
+
+  getBucketSourceScope(source: BucketDataSource): BucketDataScope {
+    const syncConfig = this.bucketDataSourceSyncConfig.get(source);
+    if (syncConfig == null) {
+      throw new ServiceAssertionError(`No sync config found for bucket data source ${source.uniqueName}`);
+    }
+    const mapping = syncConfig.mapping;
+    const defId = mapping.bucketSourceId(source);
+    // Currently uniqueName is constant for a BucketDataSource within a replication stream. This may change in the
+    // future, in which case we need to reconsider the bucketPrefix here.
+    // However, the definition may change without changing the name, so we include the defId in the bucketPrefix.
+    // defId is not unique across replication streams, so we include the replicationStreamId as well.
+    return {
+      bucketPrefix: `${source.uniqueName}.${this.replicationStreamId.toString(16)}.${defId}`,
+      source
+    };
+  }
+
+  getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope {
+    const syncConfig = this.parameterIndexLookupSyncConfig.get(source);
+    if (syncConfig == null) {
+      throw new ServiceAssertionError(
+        `No sync config found for parameter index lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId}`
+      );
+    }
+    const mapping = syncConfig.mapping;
+    const defId = mapping.parameterLookupId(source);
+    return {
+      lookupName: defId,
+      queryId: '',
+      source
+    };
+  }
+}

package/src/storage/implementation/MongoParameterCompactor.ts

@@ -18,12 +18,13 @@ type ParameterCompactionReadDocument = {
  *
  * For background, see the `/docs/parameters-lookups.md` file.
  */
-export abstract class MongoParameterCompactor {
+export class MongoParameterCompactor {
   constructor(
     protected readonly db: VersionedPowerSyncMongo,
     protected readonly group_id: number,
     protected readonly checkpoint: InternalOpId,
-    protected readonly options: CompactOptions
+    protected readonly options: CompactOptions,
+    protected readonly getCollectionsCb?: () => Promise<mongo.Collection<mongo.Document>[]>
   ) {}
 
   async compact() {
@@ -33,11 +34,27 @@ export abstract class MongoParameterCompactor {
     }
   }
 
-  protected abstract getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+  protected async getCollections(): Promise<mongo.Collection<mongo.Document>[]> {
+    if (this.getCollectionsCb == null) {
+      throw new Error('getCollections callback not provided');
+    }
+    const collections = await this.getCollectionsCb();
+    // Cast from the version-specific collection type to the generic Document type
+    // used by the parameter compactor base class.
+    return collections.map((collection) => collection as unknown as mongo.Collection<mongo.Document>);
+  }
 
-  protected abstract collectionFilter(): mongo.Document;
+  protected collectionFilter(): mongo.Document {
+    return {};
+  }
 
-  protected abstract deleteFilter(doc: mongo.Document): mongo.Document;
+  protected deleteFilter(doc: mongo.Document): mongo.Document {
+    return {
+      lookup: doc.lookup,
+      _id: { $lte: doc._id },
+      key: doc.key
+    };
+  }
 
   protected async compactCollection(collection: mongo.Collection<mongo.Document>) {
     // This is the currently-active checkpoint.
@@ -68,6 +85,7 @@ export abstract class MongoParameterCompactor {
 
     const flush = async (force: boolean) => {
       if (removeIds.length >= 1000 || (force && removeIds.length > 0)) {
+        // MongoDB Filter<T> doesn't fully match our dynamic delete filter shape here.
         const results = await collection.deleteMany({ _id: { $in: removeIds } } as any);
         logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`);
         removeIds = [];
@@ -81,6 +99,7 @@ export abstract class MongoParameterCompactor {
     };
 
     while (await cursor.hasNext()) {
+      // readBufferedDocuments returns a generic type; we know the shape from our projection.
       const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[];
       checkedEntries += batch.length;
       const now = Date.now();

package/src/storage/implementation/MongoParsedSyncConfigSet.ts

@@ -0,0 +1,88 @@
+import * as sqlite from 'node:sqlite';
+
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  BucketDefinitionMapping,
+  MultiSyncConfigBucketDefinitionMapping,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage,
+  SyncConfigWithMapping,
+  SyncConfigWithRequiredMapping
+} from '@powersync/service-core';
+import {
+  CompatibilityOption,
+  DEFAULT_HYDRATION_STATE,
+  HydratedSyncConfig,
+  HydrationState,
+  nodeSqlite,
+  SyncConfigWithErrors,
+  versionedHydrationState
+} from '@powersync/service-sync-rules';
+import { StorageConfig } from './models.js';
+import { MongoHydrationState } from './MongoHydrationState.js';
+
+export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet {
+  public readonly hydrationState: HydrationState;
+  public readonly syncConfigs: SyncConfigWithErrors[];
+  public readonly replicationStreamName: string;
+  public readonly mapping: BucketDefinitionMapping;
+
+  constructor(
+    public readonly replicationStreamId: number,
+    storageConfig: StorageConfig,
+    slotName: string,
+    syncConfigs: SyncConfigWithMapping[]
+  ) {
+    this.replicationStreamName = slotName;
+    this.syncConfigs = syncConfigs.map((config) => config.syncConfig);
+    if (this.syncConfigs.length == 0) {
+      throw new ServiceAssertionError(`At least one sync config is required`);
+    }
+    const [firstConfig] = this.syncConfigs;
+    const compatibility = firstConfig.config.compatibility;
+    for (const config of this.syncConfigs) {
+      if (config.config.compatibility.equals(compatibility)) {
+        continue;
+      }
+      throw new ServiceAssertionError(
+        `All sync configs in a replication stream must use the same compatibility options`
+      );
+    }
+
+    if (storageConfig.incrementalReprocessing) {
+      if (syncConfigs.some((c) => c.mapping == null)) {
+        throw new ServiceAssertionError(`mapping is required for v3 storage`);
+      }
+      const mappedConfigs = syncConfigs as SyncConfigWithRequiredMapping[];
+      this.hydrationState = new MongoHydrationState(mappedConfigs, this.replicationStreamId);
+      this.mapping = new MultiSyncConfigBucketDefinitionMapping(mappedConfigs);
+    } else if (!compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && !storageConfig.versionedBuckets) {
+      const [syncConfig] = syncConfigs;
+      if (syncConfigs.length != 1 || syncConfig == null) {
+        throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
+      }
+      this.hydrationState = DEFAULT_HYDRATION_STATE;
+      this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
+    } else {
+      const [syncConfig] = syncConfigs;
+      if (syncConfigs.length != 1 || syncConfig == null) {
+        throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`);
+      }
+      this.hydrationState = versionedHydrationState(this.replicationStreamId);
+      this.mapping = syncConfig.mapping ?? new SingleSyncConfigBucketDefinitionMapping();
+    }
+  }
+
+  #hydratedSyncConfig: HydratedSyncConfig | undefined;
+
+  get hydratedSyncConfig(): HydratedSyncConfig {
+    this.#hydratedSyncConfig ??= new HydratedSyncConfig({
+      definitions: this.syncConfigs.map((config) => config.config),
+      createParams: {
+        hydrationState: this.hydrationState,
+        sqlite: nodeSqlite(sqlite)
+      }
+    });
+    return this.#hydratedSyncConfig;
+  }
+}

package/src/storage/implementation/MongoPersistedReplicationStream.ts

@@ -0,0 +1,118 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { ReplicationStreamStorageIds, SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { ReplicationStreamDocumentV3, SyncConfigDefinition } from '../storage-index.js';
+import { PowerSyncMongo } from './db.js';
+import { getMongoStorageConfig } from './models.js';
+import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
+import {
+  MongoPersistedSyncConfigContentBase,
+  MongoPersistedSyncConfigContentV1,
+  MongoPersistedSyncConfigContentV3
+} from './MongoPersistedSyncConfigContent.js';
+import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
+import { SyncRuleDocumentV1 } from './v1/models.js';
+
+export class MongoPersistedReplicationStream extends storage.PersistedReplicationStream {
+  public current_lock: MongoSyncRulesLock | null = null;
+  public readonly syncConfigContent: readonly MongoPersistedSyncConfigContentBase[];
+
+  constructor(
+    private readonly db: PowerSyncMongo,
+    private readonly doc: SyncRuleDocumentV1 | ReplicationStreamDocumentV3,
+    private readonly configs: SyncConfigDefinition[] = []
+  ) {
+    const storageVersion = doc.storage_version ?? storage.LEGACY_STORAGE_VERSION;
+    const replicationJobId =
+      configs.length == 0
+        ? String(doc._id)
+        : `${doc._id}:${configs
+            .map((config) => config._id.toHexString())
+            .sort()
+            .join(',')}`;
+
+    super({
+      replicationStreamId: doc._id,
+      replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
+      state: doc.state,
+      storageVersion,
+      replicationJobId
+    });
+
+    this.syncConfigContent = this.createSyncConfigContent();
+  }
+
+  getStorageConfig() {
+    return getMongoStorageConfig(this.storageVersion);
+  }
+
+  private createSyncConfigContent(): MongoPersistedSyncConfigContentBase[] {
+    if (this.getStorageConfig().incrementalReprocessing) {
+      if (this.configs.length == 0) {
+        throw new ServiceAssertionError(`Cannot create v3 storage without sync config definitions`);
+      }
+      return this.configs.map(
+        (config) => new MongoPersistedSyncConfigContentV3(this.db, this.doc as ReplicationStreamDocumentV3, config)
+      );
+    }
+
+    return [new MongoPersistedSyncConfigContentV1(this.db, this.doc as SyncRuleDocumentV1)];
+  }
+
+  get syncConfigIds(): bson.ObjectId[] {
+    return this.configs.map((config) => config._id);
+  }
+
+  #storageIds: ReplicationStreamStorageIds | undefined;
+
+  /**
+   * Persisted storage ids for all sync configs in this replication stream.
+   *
+   * Derived from the persisted rule_mapping documents - requires no parsed sync configs.
+   */
+  get storageIds(): ReplicationStreamStorageIds {
+    this.#storageIds ??= new ReplicationStreamStorageIds(this.syncConfigContent.map((content) => content.mapping));
+    return this.#storageIds;
+  }
+
+  /**
+   * Parse the sync configs for this replication stream.
+   *
+   * This creates a fresh parse on every call. Do not call this from operational paths -
+   * use {@link MongoSyncBucketStorage.getParsedSyncConfigSet} instead, which returns a
+   * canonical instance so that parsed source objects and mappings stay associated.
+   */
+  parsed(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
+    const storageConfig = this.getStorageConfig();
+    if (!storageConfig.incrementalReprocessing) {
+      return this.syncConfigContent[0].parsed(options);
+    }
+
+    const syncConfigs = this.configs.map((config) => {
+      return {
+        syncConfigId: config._id.toHexString(),
+        syncConfig: storage.parsePersistedSyncConfigContent({
+          content: config.content,
+          compiledPlan: config.serialized_plan ?? null,
+          storageVersion: this.storageVersion,
+          parseOptions: options
+        }),
+        mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping)
+      };
+    });
+
+    return new MongoParsedSyncConfigSet(
+      this.replicationStreamId,
+      storageConfig,
+      this.replicationStreamName,
+      syncConfigs
+    );
+  }
+
+  async lock(session?: mongo.ClientSession) {
+    const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this, session);
+    this.current_lock = lock;
+    return lock;
+  }
+}