@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/src/storage/MongoBucketStorage.ts

@@ -1,22 +1,48 @@
-import { GetIntanceOptions, storage } from '@powersync/service-core';
-
-import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import {
+  GetIntanceOptions,
+  LEGACY_STORAGE_VERSION,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage
+} from '@powersync/service-core';
+
+import { DO_NOT_LOG, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 
-import { generateSlotName } from '../utils/util.js';
-import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js';
+import {
+  describeIncrementalSyncConfigUpdate,
+  formatIncrementalSyncConfigUpdateLog,
+  isCompatible
+} from '@powersync/service-core';
+import { ObjectId } from 'bson';
+import { generateReplicationStreamName } from '../utils/util.js';
 import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { PowerSyncMongo } from './implementation/db.js';
-import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js';
+import { getMongoStorageConfig, StorageConfig, SyncRuleDocumentBase } from './implementation/models.js';
 import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
-import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js';
+import { syncRuleStateUpdatePipeline } from './implementation/SyncRuleStateUpdate.js';
+import { SyncRuleDocumentV1 } from './implementation/v1/models.js';
+import { VersionedPowerSyncMongoV3 } from './implementation/v3/VersionedPowerSyncMongoV3.js';
+import { ReplicationStreamDocumentV3, SyncConfigDefinition, SyncRuleConfigStateV3 } from './storage-index.js';
 
 export interface MongoBucketStorageOptions {
-  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
+  /**
+   * Prefix for replication stream name and Postgres logical replication slot name.
+   */
+  replicationStreamNamePrefix: string;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  /**
+   * Reuse a compatible active replication stream by appending a new sync config.
+   *
+   * This currently requires source replication support. MongoDB sources can process multiple
+   * sync configs in one replication stream, but other source connectors still expect a single
+   * sync config per stream.
+   */
+  supportsMultipleSyncConfigs?: boolean;
 }
 
 export class MongoBucketStorage extends storage.BucketStorageFactory {
@@ -24,8 +50,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
-  // TODO: This is still Postgres specific and needs to be reworked
-  public readonly slot_name_prefix: string;
+  public readonly replicationStreamNamePrefix: string;
 
   private activeStorageCache: MongoSyncBucketStorage | undefined;
 
@@ -33,51 +58,54 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
   constructor(
     db: PowerSyncMongo,
-    options: {
-      slot_name_prefix: string;
-    },
-    private internalOptions?: MongoBucketStorageOptions
+    private options: MongoBucketStorageOptions
   ) {
     super();
     this.client = db.client;
     this.db = db;
     this.session = this.client.startSession();
-    this.slot_name_prefix = options.slot_name_prefix;
+    this.replicationStreamNamePrefix = options.replicationStreamNamePrefix;
   }
 
   async [Symbol.asyncDispose]() {
     // No-op
   }
 
-  getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage {
-    let { id, slot_name } = syncRules;
-    if ((typeof id as any) == 'bigint') {
-      id = Number(id);
+  getInstance(
+    replicationStream: storage.PersistedReplicationStream,
+    options?: GetIntanceOptions
+  ): MongoSyncBucketStorage {
+    if (!(replicationStream instanceof MongoPersistedReplicationStream)) {
+      throw new Error(`Expected MongoPersistedReplicationStream`);
+    }
+    let { replicationStreamId, replicationStreamName } = replicationStream;
+    if ((typeof replicationStreamId as any) == 'bigint') {
+      replicationStreamId = Number(replicationStreamId);
     }
-    const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig();
-    const storage = createMongoSyncBucketStorage(
+    const storageConfig = replicationStream.getStorageConfig();
+    const syncRuleStorage = createMongoSyncBucketStorage(
       this,
-      id,
-      syncRules as MongoPersistedSyncRulesContent,
-      slot_name,
+      replicationStreamId,
+      replicationStream,
+      replicationStreamName,
       undefined,
       {
-        ...this.internalOptions,
+        checksumOptions: this.options.checksumOptions,
         storageConfig
       }
     );
     if (!options?.skipLifecycleHooks) {
-      this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+      this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage));
     }
 
-    storage.registerListener({
+    syncRuleStorage.registerListener({
       batchStarted: (batch) => {
         batch.registerListener({
           replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
         });
       }
     });
-    return storage;
+    return syncRuleStorage;
   }
 
   async getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier> {
@@ -97,29 +125,25 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     };
   }
 
-  async restartReplication(sync_rules_group_id: number) {
-    const next = await this.getNextSyncRulesContent();
-    const active = await this.getActiveSyncRulesContent();
+  async restartReplication(replicationStreamId: number) {
+    const next = await this.getDeployingSyncConfig();
+    const active = await this.getActiveSyncConfig();
 
-    if (next != null && next.id == sync_rules_group_id) {
+    if (next != null && next.content.replicationStreamId == replicationStreamId) {
       // We need to redo the "next" replication stream
-      await this.updateSyncRules(next.asUpdateOptions());
+      await this.updateSyncRules(next.content.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sync_rules.updateOne(
         {
-          _id: next.id,
+          _id: next.content.replicationStreamId,
           state: storage.SyncRuleState.PROCESSING
         },
-        {
-          $set: {
-            state: storage.SyncRuleState.STOP
-          }
-        }
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP)
       );
       await this.db.notifyCheckpoint();
-    } else if (next == null && active?.id == sync_rules_group_id) {
+    } else if (next == null && active?.content.replicationStreamId == replicationStreamId) {
       // Slot removed for "active" replication stream, while there is no "next" one.
-      await this.updateSyncRules(active.asUpdateOptions());
+      await this.updateSyncRules(active.content.asUpdateOptions());
 
       // In this case we keep the old one as active for clients, so that that existing clients
       // can still get the latest data while we replicate the new ones.
@@ -127,48 +151,340 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
 
       await this.db.sync_rules.updateOne(
         {
-          _id: active.id,
+          _id: active.content.replicationStreamId,
           state: storage.SyncRuleState.ACTIVE
         },
-        {
-          $set: {
-            state: storage.SyncRuleState.ERRORED
-          }
-        }
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED)
       );
       await this.db.notifyCheckpoint();
-    } else if (next != null && active?.id == sync_rules_group_id) {
+    } else if (next != null && active?.content.replicationStreamId == replicationStreamId) {
       // Already have next replication stream, but need to stop replicating the active one.
 
       await this.db.sync_rules.updateOne(
         {
-          _id: active.id,
+          _id: active.content.replicationStreamId,
           state: storage.SyncRuleState.ACTIVE
         },
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED)
+      );
+      await this.db.notifyCheckpoint();
+    }
+  }
+
+  private async updateSyncRulesV3(
+    options: storage.UpdateSyncRulesOptions,
+    storageVersion: number,
+    storageConfig: StorageConfig
+  ): Promise<MongoPersistedReplicationStream> {
+    let rules: MongoPersistedReplicationStream | undefined = undefined;
+    const versioned = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3;
+
+    const session = this.session;
+
+    await session.withTransaction(async () => {
+      const active = await this.db.sync_rules.findOne<ReplicationStreamDocumentV3>(
         {
-          $set: {
-            state: storage.SyncRuleState.ERRORED
+          state: storage.SyncRuleState.ACTIVE,
+          storage_version: storageVersion
+        },
+        { session, sort: { _id: -1 }, limit: 1 }
+      );
+      if (active != null) {
+        const existingConfigDocs = await this.loadSyncConfigDefinitions(versioned, active, session);
+
+        if (
+          this.options.supportsMultipleSyncConfigs &&
+          isCompatible(
+            existingConfigDocs.map((d) => d.serialized_plan ?? null),
+            options.config,
+            logger
+          )
+        ) {
+          logger.info(`Using incremental reprocessing`);
+          await this.db.sync_rules.updateMany(
+            {
+              state: storage.SyncRuleState.PROCESSING
+            },
+            syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP),
+            { session }
+          );
+          await this.stopEmbeddedDeployingConfigs(active, session);
+          rules = await this.appendSyncConfigToStream({
+            versioned,
+            existing: active,
+            existingConfigDocs,
+            options,
+            storageVersion,
+            session
+          });
+          return;
+        }
+
+        await this.stopEmbeddedDeployingConfigs(active, session);
+      }
+
+      // Only have a single replication stream with PROCESSING.
+      await this.db.sync_rules.updateMany(
+        {
+          state: storage.SyncRuleState.PROCESSING
+        },
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP),
+        { session }
+      );
+
+      const id_doc = await this.db.op_id_sequence.findOneAndUpdate(
+        {
+          _id: 'sync_rules'
+        },
+        {
+          $inc: {
+            op_id: 1n
           }
+        },
+        {
+          upsert: true,
+          returnDocument: 'after',
+          session
         }
       );
-      await this.db.notifyCheckpoint();
+
+      const id = Number(id_doc!.op_id);
+      const replicationStreamName = generateReplicationStreamName(this.replicationStreamNamePrefix, id);
+
+      const mapping =
+        options.config.plan == null
+          ? // For legacy sync rules and streams, use the parsed config directly to create a mapping
+            SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(options.config.parsed)
+          : // For new sync streams, always use the serialized version
+            SingleSyncConfigBucketDefinitionMapping.constructIncrementalMappingFromSerializedPlans(
+              [],
+              options.config.plan.plan,
+              []
+            );
+
+      const syncConfigDoc: SyncConfigDefinition = {
+        _id: new ObjectId(),
+        replication_stream_id: id,
+        created_at: new Date(),
+        storage_version: storageVersion,
+        content: options.config.yaml,
+        serialized_plan: options.config.plan,
+        rule_mapping: mapping.serialize()
+      };
+      await versioned.syncConfigDefinitions.insertOne(syncConfigDoc, { session });
+
+      const doc: ReplicationStreamDocumentV3 = {
+        _id: id,
+        storage_version: storageVersion,
+        sync_configs: [
+          {
+            _id: syncConfigDoc._id,
+            state: storage.SyncRuleState.PROCESSING,
+            last_checkpoint: null,
+            last_checkpoint_lsn: null,
+            no_checkpoint_before: null,
+            snapshot_done: false
+          }
+        ],
+        snapshot_lsn: undefined,
+        state: storage.SyncRuleState.PROCESSING,
+        slot_name: replicationStreamName,
+        last_checkpoint_ts: null,
+        last_fatal_error: null,
+        last_fatal_error_ts: null,
+        last_keepalive_ts: null
+      };
+
+      await this.db.sync_rules.insertOne(doc, { session });
+      rules = new MongoPersistedReplicationStream(this.db, doc, [syncConfigDoc]);
+      if (options.lock) {
+        // The lock is persisted on rules.current_lock
+        await rules.lock(session);
+      }
+    });
+
+    // Notify only after the transaction has committed, so listeners cannot observe pre-commit state.
+    await this.db.notifyCheckpoint();
+
+    return rules!;
+  }
+
+  private async loadSyncConfigDefinitions(
+    versioned: VersionedPowerSyncMongoV3,
+    existing: ReplicationStreamDocumentV3,
+    session: mongo.ClientSession
+  ) {
+    const activeConfigIds = existing.sync_configs
+      .filter((config) => config.state == storage.SyncRuleState.ACTIVE)
+      .map((config) => config._id);
+
+    return versioned.syncConfigDefinitions
+      .find(
+        {
+          _id: { $in: activeConfigIds }
+        },
+        { session }
+      )
+      .toArray();
+  }
+
+  /**
+   * Load _all_ definition mappings for a replication stream - used as a base to generate new ids.
+   */
+  private async loadHistoricalSyncConfigRuleMappings(
+    versioned: VersionedPowerSyncMongoV3,
+    replicationStreamId: number,
+    session: mongo.ClientSession
+  ) {
+    return versioned.syncConfigDefinitions
+      .find(
+        {
+          replication_stream_id: replicationStreamId
+        },
+        {
+          session,
+          projection: {
+            rule_mapping: 1
+          }
+        }
+      )
+      .toArray();
+  }
+
+  private async stopEmbeddedDeployingConfigs(existing: ReplicationStreamDocumentV3, session: mongo.ClientSession) {
+    const deployingConfigs = existing.sync_configs
+      .filter((config) => config.state == storage.SyncRuleState.PROCESSING)
+      .map((config) => config._id);
+    if (deployingConfigs.length == 0) {
+      return;
+    }
+
+    await this.db.sync_rules.updateOne(
+      {
+        _id: existing._id,
+        'sync_configs._id': { $in: deployingConfigs }
+      },
+      {
+        $set: {
+          'sync_configs.$[config].state': storage.SyncRuleState.STOP
+        }
+      },
+      {
+        session,
+        arrayFilters: [{ 'config._id': { $in: deployingConfigs } }]
+      }
+    );
+  }
+
+  private logIncrementalDefinitionChanges(changes: ReturnType<typeof describeIncrementalSyncConfigUpdate>) {
+    logger.info(`Incremental reprocessing sync config update:\n${formatIncrementalSyncConfigUpdateLog(changes)}`);
+  }
+
+  private async appendSyncConfigToStream(options: {
+    versioned: VersionedPowerSyncMongoV3;
+    existing: ReplicationStreamDocumentV3;
+    existingConfigDocs: SyncConfigDefinition[];
+    options: storage.UpdateSyncRulesOptions;
+    storageVersion: number;
+    session: mongo.ClientSession;
+  }): Promise<MongoPersistedReplicationStream> {
+    const { versioned, existing, existingConfigDocs, options: updateOptions, storageVersion, session } = options;
+    const compatibleConfigs = existingConfigDocs.map((doc) => ({
+      plan: doc.serialized_plan!.plan,
+      mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)
+    }));
+    const historicalRuleMappings = await this.loadHistoricalSyncConfigRuleMappings(versioned, existing._id, session);
+    const reservedMappings = historicalRuleMappings.map((doc) =>
+      SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)
+    );
+    const mappingResult = SingleSyncConfigBucketDefinitionMapping.constructIncrementalMappingWithChanges(
+      compatibleConfigs,
+      updateOptions.config.plan!.plan,
+      reservedMappings
+    );
+    const mapping = mappingResult.mapping;
+    this.logIncrementalDefinitionChanges(
+      describeIncrementalSyncConfigUpdate({
+        activeMappings: existingConfigDocs.map((doc) =>
+          SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)
+        ),
+        newMapping: mapping,
+        newSyncConfig: updateOptions.config.parsed,
+        mappingChanges: mappingResult.changes
+      })
+    );
+
+    const syncConfigDoc: SyncConfigDefinition = {
+      _id: new ObjectId(),
+      replication_stream_id: existing._id,
+      created_at: new Date(),
+      storage_version: storageVersion,
+      content: updateOptions.config.yaml,
+      serialized_plan: updateOptions.config.plan,
+      rule_mapping: mapping.serialize()
+    };
+    await versioned.syncConfigDefinitions.insertOne(syncConfigDoc, { session });
+    const syncConfigState: SyncRuleConfigStateV3 = {
+      _id: syncConfigDoc._id,
+      state: storage.SyncRuleState.PROCESSING,
+      last_checkpoint: null,
+      last_checkpoint_lsn: null,
+      no_checkpoint_before: null,
+      snapshot_done: false
+    };
+
+    await this.db.sync_rules.updateOne(
+      { _id: existing._id },
+      {
+        $push: {
+          sync_configs: syncConfigState
+        },
+        $set: {
+          last_fatal_error: null,
+          last_fatal_error_ts: null
+        }
+      },
+      { session }
+    );
+    const syncConfigStates = [
+      ...existing.sync_configs.filter((config) => config.state == storage.SyncRuleState.ACTIVE),
+      syncConfigState
+    ];
+    const stream = new MongoPersistedReplicationStream(
+      this.db,
+      {
+        ...existing,
+        sync_configs: syncConfigStates
+      },
+      [...existingConfigDocs, syncConfigDoc]
+    );
+    if (updateOptions.lock) {
+      await stream.lock(session);
     }
+    return stream;
   }
 
-  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
+  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedReplicationStream> {
     const storageVersion =
       options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+
     const storageConfig = getMongoStorageConfig(storageVersion);
+    if (storageConfig.incrementalReprocessing) {
+      return this.updateSyncRulesV3(options, storageVersion, storageConfig);
+    }
 
-    let rules: MongoPersistedSyncRulesContent | undefined = undefined;
+    let rules: MongoPersistedReplicationStream | undefined = undefined;
 
-    await this.session.withTransaction(async () => {
+    const session = this.session;
+
+    await session.withTransaction(async () => {
       // Only have a single replication stream with PROCESSING.
       await this.db.sync_rules.updateMany(
         {
           state: storage.SyncRuleState.PROCESSING
         },
-        { $set: { state: storage.SyncRuleState.STOP } }
+        syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP),
+        { session }
       );
 
       const id_doc = await this.db.op_id_sequence.findOneAndUpdate(
@@ -182,14 +498,15 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
         },
         {
           upsert: true,
-          returnDocument: 'after'
+          returnDocument: 'after',
+          session
         }
       );
 
       const id = Number(id_doc!.op_id);
-      const slot_name = generateSlotName(this.slot_name_prefix, id);
+      const slot_name = generateReplicationStreamName(this.replicationStreamNamePrefix, id);
 
-      const doc: SyncRuleDocument = {
+      const doc: SyncRuleDocumentV1 = {
         _id: id,
         storage_version: storageVersion,
         content: options.config.yaml,
@@ -207,89 +524,141 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
         last_fatal_error_ts: null,
         last_keepalive_ts: null
       };
-      if (storageConfig.incrementalReprocessing) {
-        const parsed = options.config.parsed;
-        doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize();
-      }
-      await this.db.sync_rules.insertOne(doc);
+
+      await this.db.sync_rules.insertOne(doc, { session });
       await this.db.notifyCheckpoint();
-      rules = new MongoPersistedSyncRulesContent(this.db, doc);
+      rules = new MongoPersistedReplicationStream(this.db, doc);
       if (options.lock) {
-        const lock = await rules.lock();
+        // The lock is persisted on rules.current_lock
+        await rules.lock(session);
       }
     });
 
     return rules!;
   }
 
-  async getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
+  async getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
     const doc = await this.db.sync_rules.findOne(
       {
         state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
       },
       { sort: { _id: -1 }, limit: 1 }
     );
+
+    return this.resolvedSyncConfigFromDoc(doc, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED], {
+      cacheActiveStorage: true
+    });
+  }
+
+  private async replicationStreamFromDoc(doc: SyncRuleDocumentBase | null, stateFilter: storage.SyncRuleState[]) {
     if (doc == null) {
       return null;
     }
+    const storageConfig = getMongoStorageConfig(doc.storage_version ?? LEGACY_STORAGE_VERSION);
+
+    if (storageConfig.incrementalReprocessing) {
+      const v3 = doc as ReplicationStreamDocumentV3;
+      const matching = v3.sync_configs.filter((c) => stateFilter.includes(c.state));
+      if (matching.length == 0) {
+        return null;
+      }
+
+      // TODO: cache the config. It could specifically help for the main replication loop
+      // that checks for active replication streams.
+      // It is not a major bottleneck though, since it only runs once every couple of seconds at most.
+      const db = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3;
+      const syncConfigDocs = await db.syncConfigDefinitions
+        .find({
+          _id: { $in: matching.map((config) => config._id) }
+        })
+        .toArray();
+
+      if (syncConfigDocs.length == 0) {
+        return null;
+      }
+      return new MongoPersistedReplicationStream(this.db, v3, syncConfigDocs);
+    }
 
-    return new MongoPersistedSyncRulesContent(this.db, doc);
+    return new MongoPersistedReplicationStream(this.db, doc as SyncRuleDocumentV1);
   }
 
-  async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
+  async getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
     const doc = await this.db.sync_rules.findOne(
       {
-        state: storage.SyncRuleState.PROCESSING
+        $or: [{ state: storage.SyncRuleState.PROCESSING }, { 'sync_configs.state': storage.SyncRuleState.PROCESSING }]
       },
       { sort: { _id: -1 }, limit: 1 }
     );
-    if (doc == null) {
-      return null;
-    }
 
-    return new MongoPersistedSyncRulesContent(this.db, doc);
+    return this.resolvedSyncConfigFromDoc(doc, [storage.SyncRuleState.PROCESSING]);
   }
 
-  async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
+  async getReplicatingReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
     const docs = await this.db.sync_rules
       .find({
         state: { $in: [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE] }
       })
       .toArray();
 
-    return docs.map((doc) => {
-      return new MongoPersistedSyncRulesContent(this.db, doc);
-    });
+    return (
+      await Promise.all(
+        docs.map((doc) => {
+          return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]);
+        })
+      )
+    ).filter((r) => r != null);
   }
 
-  async getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
+  async getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
     const docs = await this.db.sync_rules
       .find({
         state: storage.SyncRuleState.STOP
       })
       .toArray();
 
-    return docs.map((doc) => {
-      return new MongoPersistedSyncRulesContent(this.db, doc);
-    });
+    return (
+      await Promise.all(
+        docs.map((doc) => {
+          return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.STOP]);
+        })
+      )
+    ).filter((d) => d != null);
   }
 
-  async getActiveStorage(): Promise<MongoSyncBucketStorage | null> {
-    const content = await this.getActiveSyncRulesContent();
-    if (content == null) {
+  private async resolvedSyncConfigFromDoc(
+    doc: SyncRuleDocumentBase | null,
+    stateFilter: storage.SyncRuleState[],
+    options: { cacheActiveStorage?: boolean } = {}
+  ): Promise<storage.ResolvedSyncConfig | null> {
+    const stream = await this.replicationStreamFromDoc(doc, stateFilter);
+    if (stream == null) {
       return null;
     }
 
-    // It is important that this instance is cached.
-    // Not for the instance construction itself, but to ensure that internal caches on the instance
-    // are re-used properly.
-    if (this.activeStorageCache?.group_id == content.id) {
-      return this.activeStorageCache;
-    } else {
-      const instance = this.getInstance(content);
-      this.activeStorageCache = instance;
-      return instance;
-    }
+    const content = stream.syncConfigContent[0];
+    const thisFactory = this;
+
+    return {
+      content,
+      replicationStream: stream,
+      get storage() {
+        // It is important that this instance is cached.
+        // Not for the instance construction itself, but to ensure that internal caches on the instance
+        // are re-used properly.
+        if (
+          options.cacheActiveStorage &&
+          thisFactory.activeStorageCache?.replicationStream.replicationJobId == stream.replicationJobId
+        ) {
+          return thisFactory.activeStorageCache;
+        }
+
+        const instance = thisFactory.getInstance(stream);
+        if (options.cacheActiveStorage) {
+          thisFactory.activeStorageCache = instance;
+        }
+        return instance;
+      }
+    };
   }
 
   async getStorageMetrics(): Promise<storage.StorageMetrics> {