@powersync/service-module-mongodb 0.0.0-dev-20241001150444

package/dist/replication/ChangeStream.js

@@ -0,0 +1,389 @@
+import { container, logger } from '@powersync/lib-services-framework';
+import { Metrics } from '@powersync/service-core';
+import { constructAfterRecord, createCheckpoint, getMongoLsn, getMongoRelation, mongoLsnToTimestamp } from './MongoRelation.js';
+export const ZERO_LSN = '0000000000000000';
+export class MissingReplicationSlotError extends Error {
+    constructor(message) {
+        super(message);
+    }
+}
+export class ChangeStream {
+    constructor(options) {
+        this.connection_id = 1;
+        this.relation_cache = new Map();
+        this.storage = options.storage;
+        this.group_id = options.storage.group_id;
+        this.connections = options.connections;
+        this.client = this.connections.client;
+        this.defaultDb = this.connections.db;
+        this.sync_rules = options.storage.getParsedSyncRules({
+            defaultSchema: this.defaultDb.databaseName
+        });
+        this.abort_signal = options.abort_signal;
+        this.abort_signal.addEventListener('abort', () => {
+            // TODO: Fast abort?
+        }, { once: true });
+    }
+    get stopped() {
+        return this.abort_signal.aborted;
+    }
+    async getQualifiedTableNames(batch, tablePattern) {
+        const schema = tablePattern.schema;
+        if (tablePattern.connectionTag != this.connections.connectionTag) {
+            return [];
+        }
+        let nameFilter;
+        if (tablePattern.isWildcard) {
+            nameFilter = new RegExp('^' + escapeRegExp(tablePattern.tablePrefix));
+        }
+        else {
+            nameFilter = tablePattern.name;
+        }
+        let result = [];
+        // Check if the collection exists
+        const collections = await this.client
+            .db(schema)
+            .listCollections({
+            name: nameFilter
+        }, { nameOnly: true })
+            .toArray();
+        for (let collection of collections) {
+            const table = await this.handleRelation(batch, {
+                name: collection.name,
+                schema,
+                objectId: collection.name,
+                replicationColumns: [{ name: '_id' }]
+            }, 
+            // This is done as part of the initial setup - snapshot is handled elsewhere
+            { snapshot: false });
+            result.push(table);
+        }
+        return result;
+    }
+    async initSlot() {
+        const status = await this.storage.getStatus();
+        if (status.snapshot_done && status.checkpoint_lsn) {
+            logger.info(`Initial replication already done`);
+            return { needsInitialSync: false };
+        }
+        return { needsInitialSync: true };
+    }
+    async estimatedCount(table) {
+        const db = this.client.db(table.schema);
+        const count = db.collection(table.table).estimatedDocumentCount();
+        return `~${count}`;
+    }
+    /**
+     * Start initial replication.
+     *
+     * If (partial) replication was done before on this slot, this clears the state
+     * and starts again from scratch.
+     */
+    async startInitialReplication() {
+        await this.storage.clear();
+        await this.initialReplication();
+    }
+    async initialReplication() {
+        const sourceTables = this.sync_rules.getSourceTables();
+        await this.client.connect();
+        const hello = await this.defaultDb.command({ hello: 1 });
+        const startTime = hello.lastWrite?.majorityOpTime?.ts;
+        if (hello.msg == 'isdbgrid') {
+            throw new Error('Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
+        }
+        else if (hello.setName == null) {
+            throw new Error('Standalone MongoDB instances are not supported - use a replicaset.');
+        }
+        else if (startTime == null) {
+            // Not known where this would happen apart from the above cases
+            throw new Error('MongoDB lastWrite timestamp not found.');
+        }
+        const session = await this.client.startSession({
+            snapshot: true
+        });
+        try {
+            await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName }, async (batch) => {
+                for (let tablePattern of sourceTables) {
+                    const tables = await this.getQualifiedTableNames(batch, tablePattern);
+                    for (let table of tables) {
+                        await this.snapshotTable(batch, table, session);
+                        await batch.markSnapshotDone([table], ZERO_LSN);
+                        await touch();
+                    }
+                }
+                const snapshotTime = session.clusterTime?.clusterTime ?? startTime;
+                if (snapshotTime != null) {
+                    const lsn = getMongoLsn(snapshotTime);
+                    logger.info(`Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`);
+                    // keepalive() does an auto-commit if there is data
+                    await batch.flush();
+                    await batch.keepalive(lsn);
+                }
+                else {
+                    throw new Error(`No snapshot clusterTime available.`);
+                }
+            });
+        }
+        finally {
+            session.endSession();
+        }
+    }
+    getSourceNamespaceFilters() {
+        const sourceTables = this.sync_rules.getSourceTables();
+        let $inFilters = [{ db: this.defaultDb.databaseName, coll: '_powersync_checkpoints' }];
+        let $refilters = [];
+        for (let tablePattern of sourceTables) {
+            if (tablePattern.connectionTag != this.connections.connectionTag) {
+                continue;
+            }
+            if (tablePattern.isWildcard) {
+                $refilters.push({ db: tablePattern.schema, coll: new RegExp('^' + escapeRegExp(tablePattern.tablePrefix)) });
+            }
+            else {
+                $inFilters.push({
+                    db: tablePattern.schema,
+                    coll: tablePattern.name
+                });
+            }
+        }
+        if ($refilters.length > 0) {
+            return { $or: [{ ns: { $in: $inFilters } }, ...$refilters] };
+        }
+        return { ns: { $in: $inFilters } };
+    }
+    static *getQueryData(results) {
+        for (let row of results) {
+            yield constructAfterRecord(row);
+        }
+    }
+    async snapshotTable(batch, table, session) {
+        logger.info(`Replicating ${table.qualifiedName}`);
+        const estimatedCount = await this.estimatedCount(table);
+        let at = 0;
+        const db = this.client.db(table.schema);
+        const collection = db.collection(table.table);
+        const query = collection.find({}, { session });
+        const cursor = query.stream();
+        for await (let document of cursor) {
+            if (this.abort_signal.aborted) {
+                throw new Error(`Aborted initial replication`);
+            }
+            const record = constructAfterRecord(document);
+            // This auto-flushes when the batch reaches its size limit
+            await batch.save({
+                tag: 'insert',
+                sourceTable: table,
+                before: undefined,
+                beforeReplicaId: undefined,
+                after: record,
+                afterReplicaId: document._id
+            });
+            at += 1;
+            Metrics.getInstance().rows_replicated_total.add(1);
+            await touch();
+        }
+        await batch.flush();
+    }
+    async getRelation(batch, descriptor) {
+        const existing = this.relation_cache.get(descriptor.objectId);
+        if (existing != null) {
+            return existing;
+        }
+        return this.handleRelation(batch, descriptor, { snapshot: false });
+    }
+    async handleRelation(batch, descriptor, options) {
+        const snapshot = options.snapshot;
+        if (!descriptor.objectId && typeof descriptor.objectId != 'string') {
+            throw new Error('objectId expected');
+        }
+        const result = await this.storage.resolveTable({
+            group_id: this.group_id,
+            connection_id: this.connection_id,
+            connection_tag: this.connections.connectionTag,
+            entity_descriptor: descriptor,
+            sync_rules: this.sync_rules
+        });
+        this.relation_cache.set(descriptor.objectId, result.table);
+        // Drop conflicting tables. This includes for example renamed tables.
+        await batch.drop(result.dropTables);
+        // Snapshot if:
+        // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
+        // 2. Snapshot is not already done, AND:
+        // 3. The table is used in sync rules.
+        const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny;
+        if (shouldSnapshot) {
+            // Truncate this table, in case a previous snapshot was interrupted.
+            await batch.truncate([result.table]);
+            await this.snapshotTable(batch, result.table);
+            const no_checkpoint_before_lsn = await createCheckpoint(this.client, this.defaultDb);
+            const [table] = await batch.markSnapshotDone([result.table], no_checkpoint_before_lsn);
+            return table;
+        }
+        return result.table;
+    }
+    async writeChange(batch, table, change) {
+        if (!table.syncAny) {
+            logger.debug(`Collection ${table.qualifiedName} not used in sync rules - skipping`);
+            return null;
+        }
+        Metrics.getInstance().rows_replicated_total.add(1);
+        if (change.operationType == 'insert') {
+            const baseRecord = constructAfterRecord(change.fullDocument);
+            return await batch.save({
+                tag: 'insert',
+                sourceTable: table,
+                before: undefined,
+                beforeReplicaId: undefined,
+                after: baseRecord,
+                afterReplicaId: change.documentKey._id
+            });
+        }
+        else if (change.operationType == 'update' || change.operationType == 'replace') {
+            if (change.fullDocument == null) {
+                // Treat as delete
+                return await batch.save({
+                    tag: 'delete',
+                    sourceTable: table,
+                    before: undefined,
+                    beforeReplicaId: change.documentKey._id
+                });
+            }
+            const after = constructAfterRecord(change.fullDocument);
+            return await batch.save({
+                tag: 'update',
+                sourceTable: table,
+                before: undefined,
+                beforeReplicaId: undefined,
+                after: after,
+                afterReplicaId: change.documentKey._id
+            });
+        }
+        else if (change.operationType == 'delete') {
+            return await batch.save({
+                tag: 'delete',
+                sourceTable: table,
+                before: undefined,
+                beforeReplicaId: change.documentKey._id
+            });
+        }
+        else {
+            throw new Error(`Unsupported operation: ${change.operationType}`);
+        }
+    }
+    async replicate() {
+        try {
+            // If anything errors here, the entire replication process is halted, and
+            // all connections automatically closed, including this one.
+            await this.initReplication();
+            await this.streamChanges();
+        }
+        catch (e) {
+            await this.storage.reportError(e);
+            throw e;
+        }
+    }
+    async initReplication() {
+        const result = await this.initSlot();
+        if (result.needsInitialSync) {
+            await this.startInitialReplication();
+        }
+    }
+    async streamChanges() {
+        // Auto-activate as soon as initial replication is done
+        await this.storage.autoActivate();
+        await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName }, async (batch) => {
+            const lastLsn = batch.lastCheckpointLsn;
+            const startAfter = mongoLsnToTimestamp(lastLsn) ?? undefined;
+            logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`);
+            // TODO: Use changeStreamSplitLargeEvent
+            const pipeline = [
+                {
+                    $match: this.getSourceNamespaceFilters()
+                }
+            ];
+            const stream = this.client.watch(pipeline, {
+                startAtOperationTime: startAfter,
+                showExpandedEvents: true,
+                useBigInt64: true,
+                maxAwaitTimeMS: 200,
+                fullDocument: 'updateLookup'
+            });
+            if (this.abort_signal.aborted) {
+                stream.close();
+                return;
+            }
+            this.abort_signal.addEventListener('abort', () => {
+                stream.close();
+            });
+            let waitForCheckpointLsn = null;
+            while (true) {
+                if (this.abort_signal.aborted) {
+                    break;
+                }
+                const changeDocument = await stream.tryNext();
+                if (changeDocument == null || this.abort_signal.aborted) {
+                    continue;
+                }
+                await touch();
+                if (startAfter != null && changeDocument.clusterTime?.lte(startAfter)) {
+                    continue;
+                }
+                // console.log('event', changeDocument);
+                if ((changeDocument.operationType == 'insert' ||
+                    changeDocument.operationType == 'update' ||
+                    changeDocument.operationType == 'replace') &&
+                    changeDocument.ns.coll == '_powersync_checkpoints') {
+                    const lsn = getMongoLsn(changeDocument.clusterTime);
+                    if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
+                        waitForCheckpointLsn = null;
+                    }
+                    await batch.flush();
+                    await batch.keepalive(lsn);
+                }
+                else if (changeDocument.operationType == 'insert' ||
+                    changeDocument.operationType == 'update' ||
+                    changeDocument.operationType == 'replace' ||
+                    changeDocument.operationType == 'delete') {
+                    if (waitForCheckpointLsn == null) {
+                        waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb);
+                    }
+                    const rel = getMongoRelation(changeDocument.ns);
+                    const table = await this.getRelation(batch, rel);
+                    if (table.syncAny) {
+                        await this.writeChange(batch, table, changeDocument);
+                    }
+                }
+                else if (changeDocument.operationType == 'drop') {
+                    const rel = getMongoRelation(changeDocument.ns);
+                    const table = await this.getRelation(batch, rel);
+                    if (table.syncAny) {
+                        await batch.drop([table]);
+                        this.relation_cache.delete(table.objectId);
+                    }
+                }
+                else if (changeDocument.operationType == 'rename') {
+                    const relFrom = getMongoRelation(changeDocument.ns);
+                    const relTo = getMongoRelation(changeDocument.to);
+                    const tableFrom = await this.getRelation(batch, relFrom);
+                    if (tableFrom.syncAny) {
+                        await batch.drop([tableFrom]);
+                        this.relation_cache.delete(tableFrom.objectId);
+                    }
+                    // Here we do need to snapshot the new table
+                    await this.handleRelation(batch, relTo, { snapshot: true });
+                }
+            }
+        });
+    }
+}
+async function touch() {
+    // FIXME: The hosted Kubernetes probe does not actually check the timestamp on this.
+    // FIXME: We need a timeout of around 5+ minutes in Kubernetes if we do start checking the timestamp,
+    // or reduce PING_INTERVAL here.
+    return container.probes.touch();
+}
+function escapeRegExp(string) {
+    // https://stackoverflow.com/a/3561711/214837
+    return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&');
+}
+//# sourceMappingURL=ChangeStream.js.map

package/dist/replication/ChangeStream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChangeStream.js","sourceRoot":"","sources":["../../src/replication/ChangeStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,OAAO,EAAgD,MAAM,yBAAyB,CAAC;AAIhG,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,WAAW,EACX,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,CAAC,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AAY3C,MAAM,OAAO,2BAA4B,SAAQ,KAAK;IACpD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;IACjB,CAAC;CACF;AAED,MAAM,OAAO,YAAY;IAgBvB,YAAY,OAA4B;QAZxC,kBAAa,GAAG,CAAC,CAAC;QAUV,mBAAc,GAAG,IAAI,GAAG,EAAwC,CAAC;QAGvE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC;YACnD,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY;SAC3C,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAChC,OAAO,EACP,GAAG,EAAE;YACH,oBAAoB;QACtB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,sBAAsB,CAC1B,KAAiC,EACjC,YAA0B;QAE1B,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QACnC,IAAI,YAAY,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE;YAChE,OAAO,EAAE,CAAC;SACX;QAED,IAAI,UAA2B,CAAC;QAChC,IAAI,YAAY,CAAC,UAAU,EAAE;YAC3B,UAAU,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;SACvE;aAAM;YACL,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC;SAChC;QACD,IAAI,MAAM,GAA0B,EAAE,CAAC;QAEvC,iCAAiC;QACjC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM;aAClC,EAAE,CAAC,MAAM,CAAC;aACV,eAAe,CACd;YACE,IAAI,EAAE,UAAU;SACjB,EACD,EAAE,QAAQ,EAAE,IAAI,EAAE,CACnB;aACA,OAAO,EAAE,CAAC;QAEb,KAAK,IAAI,UAAU,IAAI,WAAW,EAAE;YAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CACrC,KAAK,EACL;gBACE,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,MAAM;gBACN,QAAQ,EAAE,UAAU,CAAC,IAAI;gBACzB,kBAAkB,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aACZ;YAC3B,4EAA4E;YAC5E,EAAE,QAAQ,EAAE,KAAK,EAAE,CACpB,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACpB;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC9C,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,cAAc,EAAE;YACjD,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAChD,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC;SACpC;QAED,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAA0B;QAC7C,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,sBAAsB,EAAE,CAAC;QAClE,OAAO,IAAI,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC3B,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;QACvD,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAE5B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,EAAE,cAAc,EAAE,EAAqB,CAAC;QACzE,IAAI,KAAK,CAAC,GAAG,IAAI,UAAU,EAAE;YAC3B,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;SAC7G;aAAM,IAAI,KAAK,CAAC,OAAO,IAAI,IAAI,EAAE;YAChC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;SACvF;aAAM,IAAI,SAAS,IAAI,IAAI,EAAE;YAC5B,+DAA+D;YAC/D,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;SAC3D;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;YAC7C,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI;YACF,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAC3B,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,EACjE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACd,KAAK,IAAI,YAAY,IAAI,YAAY,EAAE;oBACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;oBACtE,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE;wBACxB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;wBAChD,MAAM,KAAK,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;wBAEhD,MAAM,KAAK,EAAE,CAAC;qBACf;iBACF;gBAED,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,IAAI,SAAS,CAAC;gBAEnE,IAAI,YAAY,IAAI,IAAI,EAAE;oBACxB,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;oBACtC,MAAM,CAAC,IAAI,CAAC,sBAAsB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,EAAE,CAAC,CAAC;oBACrE,mDAAmD;oBACnD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;oBACpB,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;iBAC5B;qBAAM;oBACL,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;iBACvD;YACH,CAAC,CACF,CAAC;SACH;gBAAS;YACR,OAAO,CAAC,UAAU,EAAE,CAAC;SACtB;IACH,CAAC;IAEO,yBAAyB;QAC/B,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;QAEvD,IAAI,UAAU,GAAU,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;QAC9F,IAAI,UAAU,GAAU,EAAE,CAAC;QAC3B,KAAK,IAAI,YAAY,IAAI,YAAY,EAAE;YACrC,IAAI,YAAY,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE;gBAChE,SAAS;aACV;YAED,IAAI,YAAY,CAAC,UAAU,EAAE;gBAC3B,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC;aAC9G;iBAAM;gBACL,UAAU,CAAC,IAAI,CAAC;oBACd,EAAE,EAAE,YAAY,CAAC,MAAM;oBACvB,IAAI,EAAE,YAAY,CAAC,IAAI;iBACxB,CAAC,CAAC;aACJ;SACF;QACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;SAC9D;QACD,OAAO,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,CAAC,YAAY,CAAC,OAAmC;QACtD,KAAK,IAAI,GAAG,IAAI,OAAO,EAAE;YACvB,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;SACjC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,KAAiC,EACjC,KAA0B,EAC1B,OAA6B;QAE7B,MAAM,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,EAAE,GAAG,CAAC,CAAC;QAEX,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAE9B,IAAI,KAAK,EAAE,IAAI,QAAQ,IAAI,MAAM,EAAE;YACjC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;aAChD;YAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAE9C,0DAA0D;YAC1D,MAAM,KAAK,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,QAAQ;gBACb,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,SAAS;gBACjB,eAAe,EAAE,SAAS;gBAC1B,KAAK,EAAE,MAAM;gBACb,cAAc,EAAE,QAAQ,CAAC,GAAG;aAC7B,CAAC,CAAC;YAEH,EAAE,IAAI,CAAC,CAAC;YACR,OAAO,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAEnD,MAAM,KAAK,EAAE,CAAC;SACf;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,KAAiC,EACjC,UAAkC;QAElC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC9D,IAAI,QAAQ,IAAI,IAAI,EAAE;YACpB,OAAO,QAAQ,CAAC;SACjB;QACD,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,KAAiC,EACjC,UAAkC,EAClC,OAA8B;QAE9B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,OAAO,UAAU,CAAC,QAAQ,IAAI,QAAQ,EAAE;YAClE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;SACtC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;YAC7C,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,aAAa;YAC9C,iBAAiB,EAAE,UAAU;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAE3D,qEAAqE;QACrE,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEpC,eAAe;QACf,iGAAiG;QACjG,wCAAwC;QACxC,sCAAsC;QACtC,MAAM,cAAc,GAAG,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;QAC1F,IAAI,cAAc,EAAE;YAClB,oEAAoE;YACpE,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAErC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,wBAAwB,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAErF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,wBAAwB,CAAC,CAAC;YACvF,OAAO,KAAK,CAAC;SACd;QAED,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,WAAW,CACf,KAAiC,EACjC,KAA0B,EAC1B,MAAkC;QAElC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE;YAClB,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,aAAa,oCAAoC,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;SACb;QAED,OAAO,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,MAAM,CAAC,aAAa,IAAI,QAAQ,EAAE;YACpC,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC7D,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC;gBACtB,GAAG,EAAE,QAAQ;gBACb,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,SAAS;gBACjB,eAAe,EAAE,SAAS;gBAC1B,KAAK,EAAE,UAAU;gBACjB,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,GAAG;aACvC,CAAC,CAAC;SACJ;aAAM,IAAI,MAAM,CAAC,aAAa,IAAI,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,SAAS,EAAE;YAChF,IAAI,MAAM,CAAC,YAAY,IAAI,IAAI,EAAE;gBAC/B,kBAAkB;gBAClB,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC;oBACtB,GAAG,EAAE,QAAQ;oBACb,WAAW,EAAE,KAAK;oBAClB,MAAM,EAAE,SAAS;oBACjB,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,GAAG;iBACxC,CAAC,CAAC;aACJ;YACD,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,YAAa,CAAC,CAAC;YACzD,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC;gBACtB,GAAG,EAAE,QAAQ;gBACb,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,SAAS;gBACjB,eAAe,EAAE,SAAS;gBAC1B,KAAK,EAAE,KAAK;gBACZ,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,GAAG;aACvC,CAAC,CAAC;SACJ;aAAM,IAAI,MAAM,CAAC,aAAa,IAAI,QAAQ,EAAE;YAC3C,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC;gBACtB,GAAG,EAAE,QAAQ;gBACb,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,SAAS;gBACjB,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,GAAG;aACxC,CAAC,CAAC;SACJ;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;SACnE;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI;YACF,yEAAyE;YACzE,4DAA4D;YAE5D,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;SAC5B;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrC,IAAI,MAAM,CAAC,gBAAgB,EAAE;YAC3B,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACtC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,uDAAuD;QACvD,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAElC,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC/G,MAAM,OAAO,GAAG,KAAK,CAAC,iBAAiB,CAAC;YACxC,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,uBAAuB,UAAU,EAAE,OAAO,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;YAEzE,wCAAwC;YAExC,MAAM,QAAQ,GAAqB;gBACjC;oBACE,MAAM,EAAE,IAAI,CAAC,yBAAyB,EAAE;iBACzC;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE;gBACzC,oBAAoB,EAAE,UAAU;gBAChC,kBAAkB,EAAE,IAAI;gBACxB,WAAW,EAAE,IAAI;gBACjB,cAAc,EAAE,GAAG;gBACnB,YAAY,EAAE,cAAc;aAC7B,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;gBAC7B,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO;aACR;YAED,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;gBAC/C,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,CAAC,CAAC,CAAC;YAEH,IAAI,oBAAoB,GAAkB,IAAI,CAAC;YAE/C,OAAO,IAAI,EAAE;gBACX,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;oBAC7B,MAAM;iBACP;gBAED,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;gBAE9C,IAAI,cAAc,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;oBACvD,SAAS;iBACV;gBACD,MAAM,KAAK,EAAE,CAAC;gBAEd,IAAI,UAAU,IAAI,IAAI,IAAI,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,UAAU,CAAC,EAAE;oBACrE,SAAS;iBACV;gBAED,wCAAwC;gBAExC,IACE,CAAC,cAAc,CAAC,aAAa,IAAI,QAAQ;oBACvC,cAAc,CAAC,aAAa,IAAI,QAAQ;oBACxC,cAAc,CAAC,aAAa,IAAI,SAAS,CAAC;oBAC5C,cAAc,CAAC,EAAE,CAAC,IAAI,IAAI,wBAAwB,EAClD;oBACA,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAY,CAAC,CAAC;oBACrD,IAAI,oBAAoB,IAAI,IAAI,IAAI,GAAG,IAAI,oBAAoB,EAAE;wBAC/D,oBAAoB,GAAG,IAAI,CAAC;qBAC7B;oBACD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;oBACpB,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;iBAC5B;qBAAM,IACL,cAAc,CAAC,aAAa,IAAI,QAAQ;oBACxC,cAAc,CAAC,aAAa,IAAI,QAAQ;oBACxC,cAAc,CAAC,aAAa,IAAI,SAAS;oBACzC,cAAc,CAAC,aAAa,IAAI,QAAQ,EACxC;oBACA,IAAI,oBAAoB,IAAI,IAAI,EAAE;wBAChC,oBAAoB,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;qBAC5E;oBACD,MAAM,GAAG,GAAG,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;oBAChD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;oBACjD,IAAI,KAAK,CAAC,OAAO,EAAE;wBACjB,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;qBACtD;iBACF;qBAAM,IAAI,cAAc,CAAC,aAAa,IAAI,MAAM,EAAE;oBACjD,MAAM,GAAG,GAAG,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;oBAChD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;oBACjD,IAAI,KAAK,CAAC,OAAO,EAAE;wBACjB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;wBAC1B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;qBAC5C;iBACF;qBAAM,IAAI,cAAc,CAAC,aAAa,IAAI,QAAQ,EAAE;oBACnD,MAAM,OAAO,GAAG,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;oBACpD,MAAM,KAAK,GAAG,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;oBAClD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBACzD,IAAI,SAAS,CAAC,OAAO,EAAE;wBACrB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;wBAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;qBAChD;oBACD,4CAA4C;oBAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;iBAC7D;aACF;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,KAAK,UAAU,KAAK;IAClB,oFAAoF;IACpF,qGAAqG;IACrG,gCAAgC;IAChC,OAAO,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,YAAY,CAAC,MAAc;IAClC,6CAA6C;IAC7C,OAAO,MAAM,CAAC,OAAO,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAC;AAC1D,CAAC"}

package/dist/replication/ChangeStreamReplicationJob.d.ts

@@ -0,0 +1,16 @@
+import { replication } from '@powersync/service-core';
+import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
+export interface ChangeStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions {
+    connectionFactory: ConnectionManagerFactory;
+}
+export declare class ChangeStreamReplicationJob extends replication.AbstractReplicationJob {
+    private connectionFactory;
+    private readonly connectionManager;
+    constructor(options: ChangeStreamReplicationJobOptions);
+    cleanUp(): Promise<void>;
+    keepAlive(): Promise<void>;
+    private get slotName();
+    replicate(): Promise<void>;
+    replicateLoop(): Promise<void>;
+    replicateOnce(): Promise<void>;
+}

package/dist/replication/ChangeStreamReplicationJob.js

@@ -0,0 +1,90 @@
+import { container } from '@powersync/lib-services-framework';
+import { MissingReplicationSlotError, ChangeStream } from './ChangeStream.js';
+import { replication } from '@powersync/service-core';
+import * as mongo from 'mongodb';
+export class ChangeStreamReplicationJob extends replication.AbstractReplicationJob {
+    constructor(options) {
+        super(options);
+        this.connectionFactory = options.connectionFactory;
+        this.connectionManager = this.connectionFactory.create();
+    }
+    async cleanUp() {
+        // TODO: Implement?
+    }
+    async keepAlive() {
+        // TODO: Implement?
+    }
+    get slotName() {
+        return this.options.storage.slot_name;
+    }
+    async replicate() {
+        try {
+            await this.replicateLoop();
+        }
+        catch (e) {
+            // Fatal exception
+            container.reporter.captureException(e, {
+                metadata: {}
+            });
+            this.logger.error(`Replication failed`, e);
+            if (e instanceof MissingReplicationSlotError) {
+                // This stops replication on this slot, and creates a new slot
+                await this.options.storage.factory.slotRemoved(this.slotName);
+            }
+        }
+        finally {
+            this.abortController.abort();
+        }
+    }
+    async replicateLoop() {
+        while (!this.isStopped) {
+            await this.replicateOnce();
+            if (!this.isStopped) {
+                await new Promise((resolve) => setTimeout(resolve, 5000));
+            }
+        }
+    }
+    async replicateOnce() {
+        // New connections on every iteration (every error with retry),
+        // otherwise we risk repeating errors related to the connection,
+        // such as caused by cached PG schemas.
+        const connectionManager = this.connectionFactory.create();
+        try {
+            await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
+            if (this.isStopped) {
+                return;
+            }
+            const stream = new ChangeStream({
+                abort_signal: this.abortController.signal,
+                storage: this.options.storage,
+                connections: connectionManager
+            });
+            await stream.replicate();
+        }
+        catch (e) {
+            if (this.abortController.signal.aborted) {
+                return;
+            }
+            this.logger.error(`Replication error`, e);
+            if (e.cause != null) {
+                // Without this additional log, the cause may not be visible in the logs.
+                this.logger.error(`cause`, e.cause);
+            }
+            if (e instanceof mongo.MongoError && e.hasErrorLabel('NonResumableChangeStreamError')) {
+                throw new MissingReplicationSlotError(e.message);
+            }
+            else {
+                // Report the error if relevant, before retrying
+                container.reporter.captureException(e, {
+                    metadata: {}
+                });
+                // This sets the retry delay
+                this.rateLimiter?.reportError(e);
+            }
+        }
+        finally {
+            await connectionManager.end();
+        }
+    }
+}
+//# sourceMappingURL=ChangeStreamReplicationJob.js.map

package/dist/replication/ChangeStreamReplicationJob.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChangeStreamReplicationJob.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,mCAAmC,CAAC;AAE9D,OAAO,EAAE,2BAA2B,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AAMjC,MAAM,OAAO,0BAA2B,SAAQ,WAAW,CAAC,sBAAsB;IAIhF,YAAY,OAA0C;QACpD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,OAAO;QACX,mBAAmB;IACrB,CAAC;IAED,KAAK,CAAC,SAAS;QACb,mBAAmB;IACrB,CAAC;IAED,IAAY,QAAQ;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI;YACF,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;SAC5B;QAAC,OAAO,CAAC,EAAE;YACV,kBAAkB;YAClB,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;gBACrC,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YAE3C,IAAI,CAAC,YAAY,2BAA2B,EAAE;gBAC5C,8DAA8D;gBAC9D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;aAC/D;SACF;gBAAS;YACR,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;SAC9B;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;YACtB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;gBACnB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;aAC3D;SACF;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC1D,IAAI;YACF,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClB,OAAO;aACR;YACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;gBAC9B,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;SAC1B;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE;gBACvC,OAAO;aACR;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;YAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE;gBACnB,yEAAyE;gBACzE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;aACrC;YACD,IAAI,CAAC,YAAY,KAAK,CAAC,UAAU,IAAI,CAAC,CAAC,aAAa,CAAC,+BAA+B,CAAC,EAAE;gBACrF,MAAM,IAAI,2BAA2B,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;aAClD;iBAAM;gBACL,gDAAgD;gBAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;gBACH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;aAClC;SACF;gBAAS;YACR,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;SAC/B;IACH,CAAC;CACF"}

package/dist/replication/ChangeStreamReplicator.d.ts

@@ -0,0 +1,13 @@
+import { storage, replication } from '@powersync/service-core';
+import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js';
+import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
+export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
+    connectionFactory: ConnectionManagerFactory;
+}
+export declare class ChangeStreamReplicator extends replication.AbstractReplicator<ChangeStreamReplicationJob> {
+    private readonly connectionFactory;
+    constructor(options: WalStreamReplicatorOptions);
+    createJob(options: replication.CreateJobOptions): ChangeStreamReplicationJob;
+    cleanUp(syncRulesStorage: storage.SyncRulesBucketStorage): Promise<void>;
+    stop(): Promise<void>;
+}

package/dist/replication/ChangeStreamReplicator.js

@@ -0,0 +1,26 @@
+import { replication } from '@powersync/service-core';
+import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js';
+import { MongoErrorRateLimiter } from './MongoErrorRateLimiter.js';
+export class ChangeStreamReplicator extends replication.AbstractReplicator {
+    constructor(options) {
+        super(options);
+        this.connectionFactory = options.connectionFactory;
+    }
+    createJob(options) {
+        return new ChangeStreamReplicationJob({
+            id: this.createJobId(options.storage.group_id),
+            storage: options.storage,
+            connectionFactory: this.connectionFactory,
+            lock: options.lock,
+            rateLimiter: new MongoErrorRateLimiter()
+        });
+    }
+    async cleanUp(syncRulesStorage) {
+        // TODO: Implement anything?
+    }
+    async stop() {
+        await super.stop();
+        await this.connectionFactory.shutdown();
+    }
+}
+//# sourceMappingURL=ChangeStreamReplicator.js.map

package/dist/replication/ChangeStreamReplicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChangeStreamReplicator.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAE7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AAMnE,MAAM,OAAO,sBAAuB,SAAQ,WAAW,CAAC,kBAA8C;IAGpG,YAAY,OAAmC;QAC7C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrD,CAAC;IAED,SAAS,CAAC,OAAqC;QAC7C,OAAO,IAAI,0BAA0B,CAAC;YACpC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC9C,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,IAAI,qBAAqB,EAAE;SACzC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,gBAAgD;QAC5D,4BAA4B;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC;CACF"}

package/dist/replication/ConnectionManagerFactory.d.ts

@@ -0,0 +1,9 @@
+import { NormalizedMongoConnectionConfig } from '../types/types.js';
+import { MongoManager } from './MongoManager.js';
+export declare class ConnectionManagerFactory {
+    private readonly connectionManagers;
+    private readonly dbConnectionConfig;
+    constructor(dbConnectionConfig: NormalizedMongoConnectionConfig);
+    create(): MongoManager;
+    shutdown(): Promise<void>;
+}

package/dist/replication/ConnectionManagerFactory.js

@@ -0,0 +1,21 @@
+import { logger } from '@powersync/lib-services-framework';
+import { MongoManager } from './MongoManager.js';
+export class ConnectionManagerFactory {
+    constructor(dbConnectionConfig) {
+        this.dbConnectionConfig = dbConnectionConfig;
+        this.connectionManagers = [];
+    }
+    create() {
+        const manager = new MongoManager(this.dbConnectionConfig);
+        this.connectionManagers.push(manager);
+        return manager;
+    }
+    async shutdown() {
+        logger.info('Shutting down MongoDB connection Managers...');
+        for (const manager of this.connectionManagers) {
+            await manager.end();
+        }
+        logger.info('MongoDB connection Managers shutdown completed.');
+    }
+}
+//# sourceMappingURL=ConnectionManagerFactory.js.map

package/dist/replication/ConnectionManagerFactory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConnectionManagerFactory.js","sourceRoot":"","sources":["../../src/replication/ConnectionManagerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,OAAO,wBAAwB;IAInC,YAAY,kBAAmD;QAC7D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;IAC/B,CAAC;IAED,MAAM;QACJ,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1D,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC5D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,kBAAkB,EAAE;YAC7C,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;SACrB;QACD,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IACjE,CAAC;CACF"}

package/dist/replication/MongoErrorRateLimiter.d.ts

@@ -0,0 +1,11 @@
+/// <reference types="node" resolution-mode="require"/>
+import { ErrorRateLimiter } from '@powersync/service-core';
+export declare class MongoErrorRateLimiter implements ErrorRateLimiter {
+    nextAllowed: number;
+    waitUntilAllowed(options?: {
+        signal?: AbortSignal | undefined;
+    } | undefined): Promise<void>;
+    mayPing(): boolean;
+    reportError(e: any): void;
+    private setDelay;
+}