@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/CHANGELOG.md

@@ -1,5 +1,58 @@
 # @powersync/service-module-mongodb-storage
 
+## 0.18.0
+
+### Minor Changes
+
+- a91a08f: [Experimental] Enable incremental reprocessing for MongoDB source + MongoDB storage. This includes significant changes to the v3 storage format.
+
+### Patch Changes
+
+- 101ea67: Include table names in "Cannot mark snapshot done while source tables still require snapshotting" error.
+- Updated dependencies [a91a08f]
+- Updated dependencies [184c39f]
+- Updated dependencies [c3f75df]
+- Updated dependencies [4bd35ea]
+  - @powersync/service-core@1.23.0
+  - @powersync/service-types@0.16.0
+  - @powersync/service-sync-rules@0.38.0
+  - @powersync/lib-services-framework@0.9.6
+  - @powersync/lib-service-mongodb@0.6.27
+
+## 0.17.0
+
+### Minor Changes
+
+- e2bf1ad: [Internal] rework resolveTables to handle multiple SourceTables.
+- cd5a222: [MongoDB V3 Storage] Split storage for "sync config definition" versus "replication stream status".
+- 15e2466: [MongoDB] Support snapshotting concurrently with streaming in storage v3+.
+- ebeaa3b: [Postgres] Skip storing a redundant copy of replicated rows in `current_data` for tables with `REPLICA IDENTITY FULL`.
+
+### Patch Changes
+
+- 6e2a57e: Refactor HydratedSyncConfig to support multiple SyncConfigs.
+- 92cc83b: Add the experimental `unstable_sqlite_expression_engine` sync config option to evaluate Sync Streams with SQLite.
+- Updated dependencies [17fd96b]
+- Updated dependencies [6e2a57e]
+- Updated dependencies [ec6df9f]
+- Updated dependencies [99d33d5]
+- Updated dependencies [cae92ce]
+- Updated dependencies [5ac5345]
+- Updated dependencies [15cb880]
+- Updated dependencies [f2f5086]
+- Updated dependencies [5b1b215]
+- Updated dependencies [e2bf1ad]
+- Updated dependencies [92cc83b]
+- Updated dependencies [0aab0f9]
+- Updated dependencies [15e2466]
+- Updated dependencies [ebeaa3b]
+- Updated dependencies [b116857]
+- Updated dependencies [a94b6c3]
+  - @powersync/service-core@1.22.0
+  - @powersync/service-sync-rules@0.37.0
+  - @powersync/lib-services-framework@0.9.5
+  - @powersync/lib-service-mongodb@0.6.26
+
 ## 0.16.0
 
 ### Minor Changes

package/dist/storage/MongoBucketStorage.d.ts

@@ -3,31 +3,51 @@ import { DO_NOT_LOG } from '@powersync/lib-services-framework';
 import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { PowerSyncMongo } from './implementation/db.js';
 import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
-import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js';
 export interface MongoBucketStorageOptions {
-    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
+    /**
+     * Prefix for replication stream name and Postgres logical replication slot name.
+     */
+    replicationStreamNamePrefix: string;
+    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+    /**
+     * Reuse a compatible active replication stream by appending a new sync config.
+     *
+     * This currently requires source replication support. MongoDB sources can process multiple
+     * sync configs in one replication stream, but other source connectors still expect a single
+     * sync config per stream.
+     */
+    supportsMultipleSyncConfigs?: boolean;
 }
 export declare class MongoBucketStorage extends storage.BucketStorageFactory {
-    private internalOptions?;
+    private options;
     [DO_NOT_LOG]: boolean;
     private readonly client;
     private readonly session;
-    readonly slot_name_prefix: string;
+    readonly replicationStreamNamePrefix: string;
     private activeStorageCache;
     readonly db: PowerSyncMongo;
-    constructor(db: PowerSyncMongo, options: {
-        slot_name_prefix: string;
-    }, internalOptions?: MongoBucketStorageOptions | undefined);
+    constructor(db: PowerSyncMongo, options: MongoBucketStorageOptions);
     [Symbol.asyncDispose](): Promise<void>;
-    getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage;
+    getInstance(replicationStream: storage.PersistedReplicationStream, options?: GetIntanceOptions): MongoSyncBucketStorage;
     getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier>;
-    restartReplication(sync_rules_group_id: number): Promise<void>;
-    updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent>;
-    getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null>;
-    getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null>;
-    getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
-    getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
-    getActiveStorage(): Promise<MongoSyncBucketStorage | null>;
+    restartReplication(replicationStreamId: number): Promise<void>;
+    private updateSyncRulesV3;
+    private loadSyncConfigDefinitions;
+    /**
+     * Load _all_ definition mappings for a replication stream - used as a base to generate new ids.
+     */
+    private loadHistoricalSyncConfigRuleMappings;
+    private stopEmbeddedDeployingConfigs;
+    private logIncrementalDefinitionChanges;
+    private appendSyncConfigToStream;
+    updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedReplicationStream>;
+    getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null>;
+    private replicationStreamFromDoc;
+    getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null>;
+    getReplicatingReplicationStreams(): Promise<storage.PersistedReplicationStream[]>;
+    getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]>;
+    private resolvedSyncConfigFromDoc;
     getStorageMetrics(): Promise<storage.StorageMetrics>;
     getPowerSyncInstanceId(): Promise<string>;
 }

package/dist/storage/MongoBucketStorage.js

@@ -1,53 +1,57 @@
-import { storage } from '@powersync/service-core';
-import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { LEGACY_STORAGE_VERSION, SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core';
+import { DO_NOT_LOG, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { generateSlotName } from '../utils/util.js';
-import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js';
+import { describeIncrementalSyncConfigUpdate, formatIncrementalSyncConfigUpdateLog, isCompatible } from '@powersync/service-core';
+import { ObjectId } from 'bson';
+import { generateReplicationStreamName } from '../utils/util.js';
 import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { getMongoStorageConfig } from './implementation/models.js';
-import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
+import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js';
+import { syncRuleStateUpdatePipeline } from './implementation/SyncRuleStateUpdate.js';
 export class MongoBucketStorage extends storage.BucketStorageFactory {
-    internalOptions;
+    options;
     [DO_NOT_LOG] = true;
     client;
     session;
-    // TODO: This is still Postgres specific and needs to be reworked
-    slot_name_prefix;
+    replicationStreamNamePrefix;
     activeStorageCache;
     db;
-    constructor(db, options, internalOptions) {
+    constructor(db, options) {
         super();
-        this.internalOptions = internalOptions;
+        this.options = options;
         this.client = db.client;
         this.db = db;
         this.session = this.client.startSession();
-        this.slot_name_prefix = options.slot_name_prefix;
+        this.replicationStreamNamePrefix = options.replicationStreamNamePrefix;
     }
     async [Symbol.asyncDispose]() {
         // No-op
     }
-    getInstance(syncRules, options) {
-        let { id, slot_name } = syncRules;
-        if (typeof id == 'bigint') {
-            id = Number(id);
+    getInstance(replicationStream, options) {
+        if (!(replicationStream instanceof MongoPersistedReplicationStream)) {
+            throw new Error(`Expected MongoPersistedReplicationStream`);
         }
-        const storageConfig = syncRules.getStorageConfig();
-        const storage = createMongoSyncBucketStorage(this, id, syncRules, slot_name, undefined, {
-            ...this.internalOptions,
+        let { replicationStreamId, replicationStreamName } = replicationStream;
+        if (typeof replicationStreamId == 'bigint') {
+            replicationStreamId = Number(replicationStreamId);
+        }
+        const storageConfig = replicationStream.getStorageConfig();
+        const syncRuleStorage = createMongoSyncBucketStorage(this, replicationStreamId, replicationStream, replicationStreamName, undefined, {
+            checksumOptions: this.options.checksumOptions,
             storageConfig
         });
         if (!options?.skipLifecycleHooks) {
-            this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+            this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage));
         }
-        storage.registerListener({
+        syncRuleStorage.registerListener({
             batchStarted: (batch) => {
                 batch.registerListener({
                     replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
                 });
             }
         });
-        return storage;
+        return syncRuleStorage;
     }
     async getSystemIdentifier() {
         const { setName: id } = await this.db.db.command({
@@ -61,61 +65,250 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
             type: lib_mongo.MONGO_CONNECTION_TYPE
         };
     }
-    async restartReplication(sync_rules_group_id) {
-        const next = await this.getNextSyncRulesContent();
-        const active = await this.getActiveSyncRulesContent();
-        if (next != null && next.id == sync_rules_group_id) {
+    async restartReplication(replicationStreamId) {
+        const next = await this.getDeployingSyncConfig();
+        const active = await this.getActiveSyncConfig();
+        if (next != null && next.content.replicationStreamId == replicationStreamId) {
             // We need to redo the "next" replication stream
-            await this.updateSyncRules(next.asUpdateOptions());
+            await this.updateSyncRules(next.content.asUpdateOptions());
             // Pro-actively stop replicating
             await this.db.sync_rules.updateOne({
-                _id: next.id,
+                _id: next.content.replicationStreamId,
                 state: storage.SyncRuleState.PROCESSING
-            }, {
-                $set: {
-                    state: storage.SyncRuleState.STOP
-                }
-            });
+            }, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP));
             await this.db.notifyCheckpoint();
         }
-        else if (next == null && active?.id == sync_rules_group_id) {
+        else if (next == null && active?.content.replicationStreamId == replicationStreamId) {
             // Slot removed for "active" replication stream, while there is no "next" one.
-            await this.updateSyncRules(active.asUpdateOptions());
+            await this.updateSyncRules(active.content.asUpdateOptions());
             // In this case we keep the old one as active for clients, so that that existing clients
             // can still get the latest data while we replicate the new ones.
             // It will however not replicate anymore.
             await this.db.sync_rules.updateOne({
-                _id: active.id,
+                _id: active.content.replicationStreamId,
                 state: storage.SyncRuleState.ACTIVE
-            }, {
-                $set: {
-                    state: storage.SyncRuleState.ERRORED
-                }
-            });
+            }, syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED));
             await this.db.notifyCheckpoint();
         }
-        else if (next != null && active?.id == sync_rules_group_id) {
+        else if (next != null && active?.content.replicationStreamId == replicationStreamId) {
             // Already have next replication stream, but need to stop replicating the active one.
             await this.db.sync_rules.updateOne({
-                _id: active.id,
+                _id: active.content.replicationStreamId,
                 state: storage.SyncRuleState.ACTIVE
+            }, syncRuleStateUpdatePipeline(storage.SyncRuleState.ERRORED));
+            await this.db.notifyCheckpoint();
+        }
+    }
+    async updateSyncRulesV3(options, storageVersion, storageConfig) {
+        let rules = undefined;
+        const versioned = this.db.versioned(storageConfig);
+        const session = this.session;
+        await session.withTransaction(async () => {
+            const active = await this.db.sync_rules.findOne({
+                state: storage.SyncRuleState.ACTIVE,
+                storage_version: storageVersion
+            }, { session, sort: { _id: -1 }, limit: 1 });
+            if (active != null) {
+                const existingConfigDocs = await this.loadSyncConfigDefinitions(versioned, active, session);
+                if (this.options.supportsMultipleSyncConfigs &&
+                    isCompatible(existingConfigDocs.map((d) => d.serialized_plan ?? null), options.config, logger)) {
+                    logger.info(`Using incremental reprocessing`);
+                    await this.db.sync_rules.updateMany({
+                        state: storage.SyncRuleState.PROCESSING
+                    }, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), { session });
+                    await this.stopEmbeddedDeployingConfigs(active, session);
+                    rules = await this.appendSyncConfigToStream({
+                        versioned,
+                        existing: active,
+                        existingConfigDocs,
+                        options,
+                        storageVersion,
+                        session
+                    });
+                    return;
+                }
+                await this.stopEmbeddedDeployingConfigs(active, session);
+            }
+            // Only have a single replication stream with PROCESSING.
+            await this.db.sync_rules.updateMany({
+                state: storage.SyncRuleState.PROCESSING
+            }, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), { session });
+            const id_doc = await this.db.op_id_sequence.findOneAndUpdate({
+                _id: 'sync_rules'
             }, {
-                $set: {
-                    state: storage.SyncRuleState.ERRORED
+                $inc: {
+                    op_id: 1n
                 }
+            }, {
+                upsert: true,
+                returnDocument: 'after',
+                session
             });
-            await this.db.notifyCheckpoint();
+            const id = Number(id_doc.op_id);
+            const replicationStreamName = generateReplicationStreamName(this.replicationStreamNamePrefix, id);
+            const mapping = options.config.plan == null
+                ? // For legacy sync rules and streams, use the parsed config directly to create a mapping
+                    SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(options.config.parsed)
+                : // For new sync streams, always use the serialized version
+                    SingleSyncConfigBucketDefinitionMapping.constructIncrementalMappingFromSerializedPlans([], options.config.plan.plan, []);
+            const syncConfigDoc = {
+                _id: new ObjectId(),
+                replication_stream_id: id,
+                created_at: new Date(),
+                storage_version: storageVersion,
+                content: options.config.yaml,
+                serialized_plan: options.config.plan,
+                rule_mapping: mapping.serialize()
+            };
+            await versioned.syncConfigDefinitions.insertOne(syncConfigDoc, { session });
+            const doc = {
+                _id: id,
+                storage_version: storageVersion,
+                sync_configs: [
+                    {
+                        _id: syncConfigDoc._id,
+                        state: storage.SyncRuleState.PROCESSING,
+                        last_checkpoint: null,
+                        last_checkpoint_lsn: null,
+                        no_checkpoint_before: null,
+                        snapshot_done: false
+                    }
+                ],
+                snapshot_lsn: undefined,
+                state: storage.SyncRuleState.PROCESSING,
+                slot_name: replicationStreamName,
+                last_checkpoint_ts: null,
+                last_fatal_error: null,
+                last_fatal_error_ts: null,
+                last_keepalive_ts: null
+            };
+            await this.db.sync_rules.insertOne(doc, { session });
+            rules = new MongoPersistedReplicationStream(this.db, doc, [syncConfigDoc]);
+            if (options.lock) {
+                // The lock is persisted on rules.current_lock
+                await rules.lock(session);
+            }
+        });
+        // Notify only after the transaction has committed, so listeners cannot observe pre-commit state.
+        await this.db.notifyCheckpoint();
+        return rules;
+    }
+    async loadSyncConfigDefinitions(versioned, existing, session) {
+        const activeConfigIds = existing.sync_configs
+            .filter((config) => config.state == storage.SyncRuleState.ACTIVE)
+            .map((config) => config._id);
+        return versioned.syncConfigDefinitions
+            .find({
+            _id: { $in: activeConfigIds }
+        }, { session })
+            .toArray();
+    }
+    /**
+     * Load _all_ definition mappings for a replication stream - used as a base to generate new ids.
+     */
+    async loadHistoricalSyncConfigRuleMappings(versioned, replicationStreamId, session) {
+        return versioned.syncConfigDefinitions
+            .find({
+            replication_stream_id: replicationStreamId
+        }, {
+            session,
+            projection: {
+                rule_mapping: 1
+            }
+        })
+            .toArray();
+    }
+    async stopEmbeddedDeployingConfigs(existing, session) {
+        const deployingConfigs = existing.sync_configs
+            .filter((config) => config.state == storage.SyncRuleState.PROCESSING)
+            .map((config) => config._id);
+        if (deployingConfigs.length == 0) {
+            return;
+        }
+        await this.db.sync_rules.updateOne({
+            _id: existing._id,
+            'sync_configs._id': { $in: deployingConfigs }
+        }, {
+            $set: {
+                'sync_configs.$[config].state': storage.SyncRuleState.STOP
+            }
+        }, {
+            session,
+            arrayFilters: [{ 'config._id': { $in: deployingConfigs } }]
+        });
+    }
+    logIncrementalDefinitionChanges(changes) {
+        logger.info(`Incremental reprocessing sync config update:\n${formatIncrementalSyncConfigUpdateLog(changes)}`);
+    }
+    async appendSyncConfigToStream(options) {
+        const { versioned, existing, existingConfigDocs, options: updateOptions, storageVersion, session } = options;
+        const compatibleConfigs = existingConfigDocs.map((doc) => ({
+            plan: doc.serialized_plan.plan,
+            mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)
+        }));
+        const historicalRuleMappings = await this.loadHistoricalSyncConfigRuleMappings(versioned, existing._id, session);
+        const reservedMappings = historicalRuleMappings.map((doc) => SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping));
+        const mappingResult = SingleSyncConfigBucketDefinitionMapping.constructIncrementalMappingWithChanges(compatibleConfigs, updateOptions.config.plan.plan, reservedMappings);
+        const mapping = mappingResult.mapping;
+        this.logIncrementalDefinitionChanges(describeIncrementalSyncConfigUpdate({
+            activeMappings: existingConfigDocs.map((doc) => SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(doc.rule_mapping)),
+            newMapping: mapping,
+            newSyncConfig: updateOptions.config.parsed,
+            mappingChanges: mappingResult.changes
+        }));
+        const syncConfigDoc = {
+            _id: new ObjectId(),
+            replication_stream_id: existing._id,
+            created_at: new Date(),
+            storage_version: storageVersion,
+            content: updateOptions.config.yaml,
+            serialized_plan: updateOptions.config.plan,
+            rule_mapping: mapping.serialize()
+        };
+        await versioned.syncConfigDefinitions.insertOne(syncConfigDoc, { session });
+        const syncConfigState = {
+            _id: syncConfigDoc._id,
+            state: storage.SyncRuleState.PROCESSING,
+            last_checkpoint: null,
+            last_checkpoint_lsn: null,
+            no_checkpoint_before: null,
+            snapshot_done: false
+        };
+        await this.db.sync_rules.updateOne({ _id: existing._id }, {
+            $push: {
+                sync_configs: syncConfigState
+            },
+            $set: {
+                last_fatal_error: null,
+                last_fatal_error_ts: null
+            }
+        }, { session });
+        const syncConfigStates = [
+            ...existing.sync_configs.filter((config) => config.state == storage.SyncRuleState.ACTIVE),
+            syncConfigState
+        ];
+        const stream = new MongoPersistedReplicationStream(this.db, {
+            ...existing,
+            sync_configs: syncConfigStates
+        }, [...existingConfigDocs, syncConfigDoc]);
+        if (updateOptions.lock) {
+            await stream.lock(session);
         }
+        return stream;
     }
     async updateSyncRules(options) {
         const storageVersion = options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
         const storageConfig = getMongoStorageConfig(storageVersion);
+        if (storageConfig.incrementalReprocessing) {
+            return this.updateSyncRulesV3(options, storageVersion, storageConfig);
+        }
         let rules = undefined;
-        await this.session.withTransaction(async () => {
+        const session = this.session;
+        await session.withTransaction(async () => {
             // Only have a single replication stream with PROCESSING.
             await this.db.sync_rules.updateMany({
                 state: storage.SyncRuleState.PROCESSING
-            }, { $set: { state: storage.SyncRuleState.STOP } });
+            }, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), { session });
             const id_doc = await this.db.op_id_sequence.findOneAndUpdate({
                 _id: 'sync_rules'
             }, {
@@ -124,10 +317,11 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
                 }
             }, {
                 upsert: true,
-                returnDocument: 'after'
+                returnDocument: 'after',
+                session
             });
             const id = Number(id_doc.op_id);
-            const slot_name = generateSlotName(this.slot_name_prefix, id);
+            const slot_name = generateReplicationStreamName(this.replicationStreamNamePrefix, id);
             const doc = {
                 _id: id,
                 storage_version: storageVersion,
@@ -146,73 +340,102 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
                 last_fatal_error_ts: null,
                 last_keepalive_ts: null
             };
-            if (storageConfig.incrementalReprocessing) {
-                const parsed = options.config.parsed;
-                doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize();
-            }
-            await this.db.sync_rules.insertOne(doc);
+            await this.db.sync_rules.insertOne(doc, { session });
             await this.db.notifyCheckpoint();
-            rules = new MongoPersistedSyncRulesContent(this.db, doc);
+            rules = new MongoPersistedReplicationStream(this.db, doc);
             if (options.lock) {
-                const lock = await rules.lock();
+                // The lock is persisted on rules.current_lock
+                await rules.lock(session);
             }
         });
         return rules;
     }
-    async getActiveSyncRulesContent() {
+    async getActiveSyncConfig() {
         const doc = await this.db.sync_rules.findOne({
             state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
         }, { sort: { _id: -1 }, limit: 1 });
+        return this.resolvedSyncConfigFromDoc(doc, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED], {
+            cacheActiveStorage: true
+        });
+    }
+    async replicationStreamFromDoc(doc, stateFilter) {
         if (doc == null) {
             return null;
         }
-        return new MongoPersistedSyncRulesContent(this.db, doc);
+        const storageConfig = getMongoStorageConfig(doc.storage_version ?? LEGACY_STORAGE_VERSION);
+        if (storageConfig.incrementalReprocessing) {
+            const v3 = doc;
+            const matching = v3.sync_configs.filter((c) => stateFilter.includes(c.state));
+            if (matching.length == 0) {
+                return null;
+            }
+            // TODO: cache the config. It could specifically help for the main replication loop
+            // that checks for active replication streams.
+            // It is not a major bottleneck though, since it only runs once every couple of seconds at most.
+            const db = this.db.versioned(storageConfig);
+            const syncConfigDocs = await db.syncConfigDefinitions
+                .find({
+                _id: { $in: matching.map((config) => config._id) }
+            })
+                .toArray();
+            if (syncConfigDocs.length == 0) {
+                return null;
+            }
+            return new MongoPersistedReplicationStream(this.db, v3, syncConfigDocs);
+        }
+        return new MongoPersistedReplicationStream(this.db, doc);
     }
-    async getNextSyncRulesContent() {
+    async getDeployingSyncConfig() {
         const doc = await this.db.sync_rules.findOne({
-            state: storage.SyncRuleState.PROCESSING
+            $or: [{ state: storage.SyncRuleState.PROCESSING }, { 'sync_configs.state': storage.SyncRuleState.PROCESSING }]
         }, { sort: { _id: -1 }, limit: 1 });
-        if (doc == null) {
-            return null;
-        }
-        return new MongoPersistedSyncRulesContent(this.db, doc);
+        return this.resolvedSyncConfigFromDoc(doc, [storage.SyncRuleState.PROCESSING]);
     }
-    async getReplicatingSyncRules() {
+    async getReplicatingReplicationStreams() {
         const docs = await this.db.sync_rules
             .find({
             state: { $in: [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE] }
         })
             .toArray();
-        return docs.map((doc) => {
-            return new MongoPersistedSyncRulesContent(this.db, doc);
-        });
+        return (await Promise.all(docs.map((doc) => {
+            return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]);
+        }))).filter((r) => r != null);
     }
-    async getStoppedSyncRules() {
+    async getStoppedReplicationStreams() {
         const docs = await this.db.sync_rules
             .find({
             state: storage.SyncRuleState.STOP
         })
             .toArray();
-        return docs.map((doc) => {
-            return new MongoPersistedSyncRulesContent(this.db, doc);
-        });
+        return (await Promise.all(docs.map((doc) => {
+            return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.STOP]);
+        }))).filter((d) => d != null);
     }
-    async getActiveStorage() {
-        const content = await this.getActiveSyncRulesContent();
-        if (content == null) {
+    async resolvedSyncConfigFromDoc(doc, stateFilter, options = {}) {
+        const stream = await this.replicationStreamFromDoc(doc, stateFilter);
+        if (stream == null) {
             return null;
         }
-        // It is important that this instance is cached.
-        // Not for the instance construction itself, but to ensure that internal caches on the instance
-        // are re-used properly.
-        if (this.activeStorageCache?.group_id == content.id) {
-            return this.activeStorageCache;
-        }
-        else {
-            const instance = this.getInstance(content);
-            this.activeStorageCache = instance;
-            return instance;
-        }
+        const content = stream.syncConfigContent[0];
+        const thisFactory = this;
+        return {
+            content,
+            replicationStream: stream,
+            get storage() {
+                // It is important that this instance is cached.
+                // Not for the instance construction itself, but to ensure that internal caches on the instance
+                // are re-used properly.
+                if (options.cacheActiveStorage &&
+                    thisFactory.activeStorageCache?.replicationStream.replicationJobId == stream.replicationJobId) {
+                    return thisFactory.activeStorageCache;
+                }
+                const instance = thisFactory.getInstance(stream);
+                if (options.cacheActiveStorage) {
+                    thisFactory.activeStorageCache = instance;
+                }
+                return instance;
+            }
+        };
     }
     async getStorageMetrics() {
         const ignoreNotExisting = (e) => {