@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/src/storage/MongoBucketStorage.ts

@@ -1,30 +1,55 @@
-import { GetIntanceOptions, LEGACY_STORAGE_VERSION, storage } from '@powersync/service-core';
+import {
+  GetIntanceOptions,
+  LEGACY_STORAGE_VERSION,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage
+} from '@powersync/service-core';
 
-import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { DO_NOT_LOG, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 
+import {
+  describeIncrementalSyncConfigUpdate,
+  formatIncrementalSyncConfigUpdateLog,
+  isCompatible
+} from '@powersync/service-core';
 import { ObjectId } from 'bson';
-import { generateSlotName } from '../utils/util.js';
-import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js';
+import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
+import { generateReplicationStreamName } from '../utils/util.js';
 import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { PowerSyncMongo } from './implementation/db.js';
 import { getMongoStorageConfig, StorageConfig, SyncRuleDocumentBase } from './implementation/models.js';
 import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
-import {
-  MongoPersistedSyncRulesContentV1,
-  MongoPersistedSyncRulesContentV3
-} from './implementation/MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js';
 import { syncRuleStateUpdatePipeline } from './implementation/SyncRuleStateUpdate.js';
 import { SyncRuleDocumentV1 } from './implementation/v1/models.js';
+import { ObjectStorage } from './implementation/v3/object-storage/ObjectStorage.js';
 import { VersionedPowerSyncMongoV3 } from './implementation/v3/VersionedPowerSyncMongoV3.js';
-import { ReplicationStreamDocumentV3, SyncConfigDefinition } from './storage-index.js';
+import { ReplicationStreamDocumentV3, SyncConfigDefinition, SyncRuleConfigStateV3 } from './storage-index.js';
 
 export interface MongoBucketStorageOptions {
-  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  objectStorage?: ObjectStorage;
+  inlineThresholdBytes?: number;
+  /**
+   * Prefix for replication stream name and Postgres logical replication slot name.
+   */
+  replicationStreamNamePrefix: string;
+  readPreference?: mongo.ReadPreference;
+  checksumCacheTtlMs?: number;
+  clearBatchThrottleRate?: number;
+  /**
+   * Reuse a compatible active replication stream by appending a new sync config.
+   *
+   * This currently requires source replication support. MongoDB sources can process multiple
+   * sync configs in one replication stream, but other source connectors still expect a single
+   * sync config per stream.
+   */
+  supportsMultipleSyncConfigs?: boolean;
 }
 
 export class MongoBucketStorage extends storage.BucketStorageFactory {
@@ -32,8 +57,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
-  // TODO: This is still Postgres specific and needs to be reworked
-  public readonly slot_name_prefix: string;
+  public readonly replicationStreamNamePrefix: string;
 
   private activeStorageCache: MongoSyncBucketStorage | undefined;
 
@@ -41,51 +65,59 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
   constructor(
     db: PowerSyncMongo,
-    options: {
-      slot_name_prefix: string;
-    },
-    private internalOptions?: MongoBucketStorageOptions
+    private options: MongoBucketStorageOptions
   ) {
     super();
     this.client = db.client;
     this.db = db;
     this.session = this.client.startSession();
-    this.slot_name_prefix = options.slot_name_prefix;
+    this.replicationStreamNamePrefix = options.replicationStreamNamePrefix;
   }
 
   async [Symbol.asyncDispose]() {
     // No-op
   }
 
-  getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage {
-    let { id, slot_name } = syncRules;
-    if ((typeof id as any) == 'bigint') {
-      id = Number(id);
+  getInstance(
+    replicationStream: storage.PersistedReplicationStream,
+    options?: GetIntanceOptions
+  ): MongoSyncBucketStorage {
+    if (!(replicationStream instanceof MongoPersistedReplicationStream)) {
+      throw new Error(`Expected MongoPersistedReplicationStream`);
     }
-    const storageConfig = (syncRules as MongoPersistedSyncRulesContentV1).getStorageConfig();
-    const storage = createMongoSyncBucketStorage(
+    let { replicationStreamId, replicationStreamName } = replicationStream;
+    if ((typeof replicationStreamId as any) == 'bigint') {
+      replicationStreamId = Number(replicationStreamId);
+    }
+    const storageConfig = replicationStream.getStorageConfig();
+    const syncRuleStorage = createMongoSyncBucketStorage(
       this,
-      id,
-      syncRules as MongoPersistedSyncRulesContentV1,
-      slot_name,
+      replicationStreamId,
+      replicationStream,
+      replicationStreamName,
       undefined,
       {
-        ...this.internalOptions,
-        storageConfig
+        checksumOptions: this.options.checksumOptions,
+        readPreference: this.options.readPreference,
+        checksumCacheTtlMs: this.options.checksumCacheTtlMs,
+        storageConfig,
+        objectStorage: this.options.objectStorage,
+        inlineThresholdBytes: this.options.inlineThresholdBytes,
+        clearBatchThrottleRate: this.options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE
       }
     );
     if (!options?.skipLifecycleHooks) {
-      this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+      this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage));
     }
 
-    storage.registerListener({
+    syncRuleStorage.registerListener({
       batchStarted: (batch) => {
         batch.registerListener({
           replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
         });
       }
     });
-    return storage;
+    return syncRuleStorage;
   }
 
   async getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier> {
@@ -105,25 +137,25 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     };
   }
 
-  async restartReplication(sync_rules_group_id: number) {
-    const next = await this.getNextSyncRulesContent();
-    const active = await this.getActiveSyncRulesContent();
+  async restartReplication(replicationStreamId: number) {
+    const next = await this.getDeployingSyncConfig();
+    const active = await this.getActiveSyncConfig();
 
-    if (next != null && next.id == sync_rules_group_id) {
+    if (next != null && next.content.replicationStreamId == replicationStreamId) {
       // We need to redo the "next" replication stream
-      await this.updateSyncRules(next.asUpdateOptions());
+      await this.updateSyncRules(next.content.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sync_rules.updateOne(
         {
-          _id: next.id,
+          _id: next.content.replicationStreamId,
           state: storage.SyncRuleState.PROCESSING
         },
         syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP)
       );
       await this.db.notifyCheckpoint();
-    } else if (next == null && active?.id == sync_rules_group_id) {
+    } else if (next == null && active?.content.replicationStreamId == replicationStreamId) {
       // Slot removed for "active" replication stream, while there is no "next" one.
-      await this.updateSyncRules(active.asUpdateOptions());
+      await this.updateSyncRules(active.content.asUpdateOptions());
 
       // In this case we keep the old one as active for clients, so that that existing clients
       // can still get the latest data while we replicate the new ones.
@@ -131,18 +163,18 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
       await this.db.sync_rules.updateOne(
         {
-          _id: active.id,
+          _id: active.content.replicationStreamId,
           state: storage.SyncRuleState.ACTIVE
         },
         syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED)
       );
       await this.db.notifyCheckpoint();
-    } else if (next != null && active?.id == sync_rules_group_id) {
+    } else if (next != null && active?.content.replicationStreamId == replicationStreamId) {
       // Already have next replication stream, but need to stop replicating the active one.
 
       await this.db.sync_rules.updateOne(
         {
-          _id: active.id,
+          _id: active.content.replicationStreamId,
           state: storage.SyncRuleState.ACTIVE
         },
         syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED)
@@ -155,13 +187,54 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     options: storage.UpdateSyncRulesOptions,
     storageVersion: number,
     storageConfig: StorageConfig
-  ): Promise<MongoPersistedSyncRulesContentV3> {
-    let rules: MongoPersistedSyncRulesContentV3 | undefined = undefined;
+  ): Promise<MongoPersistedReplicationStream> {
+    let rules: MongoPersistedReplicationStream | undefined = undefined;
     const versioned = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3;
 
     const session = this.session;
 
     await session.withTransaction(async () => {
+      const active = await this.db.sync_rules.findOne<ReplicationStreamDocumentV3>(
+        {
+          state: storage.SyncRuleState.ACTIVE,
+          storage_version: storageVersion
+        },
+        { session, sort: { _id: -1 }, limit: 1 }
+      );
+      if (active != null) {
+        const existingConfigDocs = await this.loadSyncConfigDefinitions(versioned, active, session);
+
+        if (
+          this.options.supportsMultipleSyncConfigs &&
+          isCompatible(
+            existingConfigDocs.map((d) => d.serialized_plan ?? null),
+            options.config,
+            logger
+          )
+        ) {
+          logger.info(`Using incremental reprocessing`);
+          await this.db.sync_rules.updateMany(
+            {
+              state: storage.SyncRuleState.PROCESSING
+            },
+            syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP),
+            { session }
+          );
+          await this.stopEmbeddedDeployingConfigs(active, session);
+          rules = await this.appendSyncConfigToStream({
+            versioned,
+            existing: active,
+            existingConfigDocs,
+            options,
+            storageVersion,
+            session
+          });
+          return;
+        }
+
+        await this.stopEmbeddedDeployingConfigs(active, session);
+      }
+
       // Only have a single replication stream with PROCESSING.
       await this.db.sync_rules.updateMany(
         {
@@ -188,9 +261,18 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       );
 
       const id = Number(id_doc!.op_id);
-      const slot_name = generateSlotName(this.slot_name_prefix, id);
-
-      const mapping = BucketDefinitionMapping.fromParsedSyncRules(options.config.parsed);
+      const replicationStreamName = generateReplicationStreamName(this.replicationStreamNamePrefix, id);
+
+      const mapping =
+        options.config.plan == null
+          ? // For legacy sync rules and streams, use the parsed config directly to create a mapping
+            SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(options.config.parsed)
+          : // For new sync streams, always use the serialized version
+            SingleSyncConfigBucketDefinitionMapping.constructIncrementalMappingFromSerializedPlans(
+              [],
+              options.config.plan.plan,
+              []
+            );
 
       const syncConfigDoc: SyncConfigDefinition = {
         _id: new ObjectId(),
@@ -210,7 +292,6 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
           {
             _id: syncConfigDoc._id,
             state: storage.SyncRuleState.PROCESSING,
-            keepalive_op: null,
             last_checkpoint: null,
             last_checkpoint_lsn: null,
             no_checkpoint_before: null,
@@ -219,7 +300,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
         ],
         snapshot_lsn: undefined,
         state: storage.SyncRuleState.PROCESSING,
-        slot_name: slot_name,
+        slot_name: replicationStreamName,
         last_checkpoint_ts: null,
         last_fatal_error: null,
         last_fatal_error_ts: null,
@@ -227,20 +308,175 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       };
 
       await this.db.sync_rules.insertOne(doc, { session });
-      await this.db.notifyCheckpoint();
-      rules = new MongoPersistedSyncRulesContentV3(this.db, doc, syncConfigDoc);
+      rules = new MongoPersistedReplicationStream(this.db, doc, [syncConfigDoc]);
       if (options.lock) {
         // The lock is persisted on rules.current_lock
         await rules.lock(session);
       }
     });
 
+    // Notify only after the transaction has committed, so listeners cannot observe pre-commit state.
+    await this.db.notifyCheckpoint();
+
     return rules!;
   }
 
-  async updateSyncRules(
-    options: storage.UpdateSyncRulesOptions
-  ): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3> {
+  private async loadSyncConfigDefinitions(
+    versioned: VersionedPowerSyncMongoV3,
+    existing: ReplicationStreamDocumentV3,
+    session: mongo.ClientSession
+  ) {
+    const activeConfigIds = existing.sync_configs
+      .filter((config) => config.state == storage.SyncRuleState.ACTIVE)
+      .map((config) => config._id);
+
+    return versioned.syncConfigDefinitions
+      .find(
+        {
+          _id: { $in: activeConfigIds }
+        },
+        { session }
+      )
+      .toArray();
+  }
+
+  /**
+   * Load _all_ definition mappings for a replication stream - used as a base to generate new ids.
+   */
+  private async loadHistoricalSyncConfigRuleMappings(
+    versioned: VersionedPowerSyncMongoV3,
+    replicationStreamId: number,
+    session: mongo.ClientSession
+  ) {
+    return versioned.syncConfigDefinitions
+      .find(
+        {
+          replication_stream_id: replicationStreamId
+        },
+        {
+          session,
+          projection: {
+            rule_mapping: 1
+          }
+        }
+      )
+      .toArray();
+  }
+
+  private async stopEmbeddedDeployingConfigs(existing: ReplicationStreamDocumentV3, session: mongo.ClientSession) {
+    const deployingConfigs = existing.sync_configs
+      .filter((config) => config.state == storage.SyncRuleState.PROCESSING)
+      .map((config) => config._id);
+    if (deployingConfigs.length == 0) {
+      return;
+    }
+
+    await this.db.sync_rules.updateOne(
+      {
+        _id: existing._id,
+        'sync_configs._id': { $in: deployingConfigs }
+      },
+      {
+        $set: {
+          'sync_configs.$[config].state': storage.SyncRuleState.STOP
+        }
+      },
+      {
+        session,
+        arrayFilters: [{ 'config._id': { $in: deployingConfigs } }]
+      }
+    );
+  }
+
+  private logIncrementalDefinitionChanges(changes: ReturnType<typeof describeIncrementalSyncConfigUpdate>) {
+    logger.info(`Incremental reprocessing sync config update:\n${formatIncrementalSyncConfigUpdateLog(changes)}`);
+  }
+
+  private async appendSyncConfigToStream(options: {
+    versioned: VersionedPowerSyncMongoV3;
+    existing: ReplicationStreamDocumentV3;
+    existingConfigDocs: SyncConfigDefinition[];
+    options: storage.UpdateSyncRulesOptions;
+    storageVersion: number;
+    session: mongo.ClientSession;
+  }): Promise<MongoPersistedReplicationStream> {
+    const { versioned, existing, existingConfigDocs, options: updateOptions, storageVersion, session } = options;
+    const compatibleConfigs = existingConfigDocs.map((doc) => ({
+      plan: doc.serialized_plan!.plan,
+      mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)
+    }));
+    const historicalRuleMappings = await this.loadHistoricalSyncConfigRuleMappings(versioned, existing._id, session);
+    const reservedMappings = historicalRuleMappings.map((doc) =>
+      SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)
+    );
+    const mappingResult = SingleSyncConfigBucketDefinitionMapping.constructIncrementalMappingWithChanges(
+      compatibleConfigs,
+      updateOptions.config.plan!.plan,
+      reservedMappings
+    );
+    const mapping = mappingResult.mapping;
+    this.logIncrementalDefinitionChanges(
+      describeIncrementalSyncConfigUpdate({
+        activeMappings: existingConfigDocs.map((doc) =>
+          SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)
+        ),
+        newMapping: mapping,
+        newSyncConfig: updateOptions.config.parsed,
+        mappingChanges: mappingResult.changes
+      })
+    );
+
+    const syncConfigDoc: SyncConfigDefinition = {
+      _id: new ObjectId(),
+      replication_stream_id: existing._id,
+      created_at: new Date(),
+      storage_version: storageVersion,
+      content: updateOptions.config.yaml,
+      serialized_plan: updateOptions.config.plan,
+      rule_mapping: mapping.serialize()
+    };
+    await versioned.syncConfigDefinitions.insertOne(syncConfigDoc, { session });
+    const syncConfigState: SyncRuleConfigStateV3 = {
+      _id: syncConfigDoc._id,
+      state: storage.SyncRuleState.PROCESSING,
+      last_checkpoint: null,
+      last_checkpoint_lsn: null,
+      no_checkpoint_before: null,
+      snapshot_done: false
+    };
+
+    await this.db.sync_rules.updateOne(
+      { _id: existing._id },
+      {
+        $push: {
+          sync_configs: syncConfigState
+        },
+        $set: {
+          last_fatal_error: null,
+          last_fatal_error_ts: null
+        }
+      },
+      { session }
+    );
+    const syncConfigStates = [
+      ...existing.sync_configs.filter((config) => config.state == storage.SyncRuleState.ACTIVE),
+      syncConfigState
+    ];
+    const stream = new MongoPersistedReplicationStream(
+      this.db,
+      {
+        ...existing,
+        sync_configs: syncConfigStates
+      },
+      [...existingConfigDocs, syncConfigDoc]
+    );
+    // The stream already exists, so an active replication job may already hold the stream lock.
+    // Deployment only persists the appended sync config; replication job locking is handled by
+    // the replicator.
+    return stream;
+  }
+
+  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedReplicationStream> {
     const storageVersion =
       options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
 
@@ -249,7 +485,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       return this.updateSyncRulesV3(options, storageVersion, storageConfig);
     }
 
-    let rules: MongoPersistedSyncRulesContentV1 | undefined = undefined;
+    let rules: MongoPersistedReplicationStream | undefined = undefined;
 
     const session = this.session;
 
@@ -280,7 +516,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       );
 
       const id = Number(id_doc!.op_id);
-      const slot_name = generateSlotName(this.slot_name_prefix, id);
+      const slot_name = generateReplicationStreamName(this.replicationStreamNamePrefix, id);
 
       const doc: SyncRuleDocumentV1 = {
         _id: id,
@@ -303,7 +539,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
       await this.db.sync_rules.insertOne(doc, { session });
       await this.db.notifyCheckpoint();
-      rules = new MongoPersistedSyncRulesContentV1(this.db, doc);
+      rules = new MongoPersistedReplicationStream(this.db, doc);
       if (options.lock) {
         // The lock is persisted on rules.current_lock
         await rules.lock(session);
@@ -313,9 +549,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     return rules!;
   }
 
-  async getActiveSyncRulesContent(): Promise<
-    MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null
-  > {
+  async getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
     const doc = await this.db.sync_rules.findOne(
       {
         state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
@@ -323,10 +557,12 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       { sort: { _id: -1 }, limit: 1 }
     );
 
-    return this.getSyncRulesContent(doc, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]);
+    return this.resolvedSyncConfigFromDoc(doc, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED], {
+      cacheActiveStorage: true
+    });
   }
 
-  private async getSyncRulesContent(doc: SyncRuleDocumentBase | null, stateFilter: storage.SyncRuleState[]) {
+  private async replicationStreamFromDoc(doc: SyncRuleDocumentBase | null, stateFilter: storage.SyncRuleState[]) {
     if (doc == null) {
       return null;
     }
@@ -334,8 +570,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
     if (storageConfig.incrementalReprocessing) {
       const v3 = doc as ReplicationStreamDocumentV3;
-      const active = v3.sync_configs.find((c) => stateFilter.includes(c.state));
-      if (active == null) {
+      const matching = v3.sync_configs.filter((c) => stateFilter.includes(c.state));
+      if (matching.length == 0) {
         return null;
       }
 
@@ -343,28 +579,33 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       // that checks for active replication streams.
       // It is not a major bottleneck though, since it only runs once every couple of seconds at most.
       const db = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3;
-      const syncConfigDoc = await db.syncConfigDefinitions.findOne({ _id: active._id });
-      if (syncConfigDoc == null) {
+      const syncConfigDocs = await db.syncConfigDefinitions
+        .find({
+          _id: { $in: matching.map((config) => config._id) }
+        })
+        .toArray();
+
+      if (syncConfigDocs.length == 0) {
         return null;
       }
-      return new MongoPersistedSyncRulesContentV3(this.db, v3, syncConfigDoc);
+      return new MongoPersistedReplicationStream(this.db, v3, syncConfigDocs);
     }
 
-    return new MongoPersistedSyncRulesContentV1(this.db, doc as SyncRuleDocumentV1);
+    return new MongoPersistedReplicationStream(this.db, doc as SyncRuleDocumentV1);
   }
 
-  async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null> {
+  async getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
     const doc = await this.db.sync_rules.findOne(
       {
-        state: storage.SyncRuleState.PROCESSING
+        $or: [{ state: storage.SyncRuleState.PROCESSING }, { 'sync_configs.state': storage.SyncRuleState.PROCESSING }]
       },
       { sort: { _id: -1 }, limit: 1 }
     );
 
-    return this.getSyncRulesContent(doc, [storage.SyncRuleState.PROCESSING]);
+    return this.resolvedSyncConfigFromDoc(doc, [storage.SyncRuleState.PROCESSING]);
   }
 
-  async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
+  async getReplicatingReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
     const docs = await this.db.sync_rules
       .find({
         state: { $in: [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE] }
@@ -374,13 +615,13 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     return (
       await Promise.all(
         docs.map((doc) => {
-          return this.getSyncRulesContent(doc, [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]);
+          return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]);
         })
       )
     ).filter((r) => r != null);
   }
 
-  async getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
+  async getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
     const docs = await this.db.sync_rules
       .find({
         state: storage.SyncRuleState.STOP
@@ -390,28 +631,46 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     return (
       await Promise.all(
         docs.map((doc) => {
-          return this.getSyncRulesContent(doc, [storage.SyncRuleState.STOP]);
+          return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.STOP]);
         })
       )
     ).filter((d) => d != null);
   }
 
-  async getActiveStorage(): Promise<MongoSyncBucketStorage | null> {
-    const content = await this.getActiveSyncRulesContent();
-    if (content == null) {
+  private async resolvedSyncConfigFromDoc(
+    doc: SyncRuleDocumentBase | null,
+    stateFilter: storage.SyncRuleState[],
+    options: { cacheActiveStorage?: boolean } = {}
+  ): Promise<storage.ResolvedSyncConfig | null> {
+    const stream = await this.replicationStreamFromDoc(doc, stateFilter);
+    if (stream == null) {
       return null;
     }
 
-    // It is important that this instance is cached.
-    // Not for the instance construction itself, but to ensure that internal caches on the instance
-    // are re-used properly.
-    if (this.activeStorageCache?.group_id == content.id) {
-      return this.activeStorageCache;
-    } else {
-      const instance = this.getInstance(content);
-      this.activeStorageCache = instance;
-      return instance;
-    }
+    const content = stream.syncConfigContent[0];
+    const thisFactory = this;
+
+    return {
+      content,
+      replicationStream: stream,
+      get storage() {
+        // It is important that this instance is cached.
+        // Not for the instance construction itself, but to ensure that internal caches on the instance
+        // are re-used properly.
+        if (
+          options.cacheActiveStorage &&
+          thisFactory.activeStorageCache?.replicationStream.replicationJobId == stream.replicationJobId
+        ) {
+          return thisFactory.activeStorageCache;
+        }
+
+        const instance = thisFactory.getInstance(stream);
+        if (options.cacheActiveStorage) {
+          thisFactory.activeStorageCache = instance;
+        }
+        return instance;
+      }
+    };
   }
 
   async getStorageMetrics(): Promise<storage.StorageMetrics> {