@powersync/service-module-mongodb 0.9.1 → 0.10.1

package/dist/replication/ChangeStream.js

@@ -1,12 +1,65 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
 import { isMongoNetworkTimeoutError, isMongoServerError, mongo } from '@powersync/lib-service-mongodb';
-import { container, DatabaseConnectionError, ErrorCode, logger, ReplicationAbortedError, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
-import { SaveOperationTag } from '@powersync/service-core';
+import { container, DatabaseConnectionError, logger as defaultLogger, ErrorCode, ReplicationAbortedError, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
+import { RelationCache, SaveOperationTag } from '@powersync/service-core';
 import { ReplicationMetric } from '@powersync/service-types';
 import { MongoLSN } from '../common/MongoLSN.js';
 import { PostImagesOption } from '../types/types.js';
 import { escapeRegExp } from '../utils.js';
 import { constructAfterRecord, createCheckpoint, getCacheIdentifier, getMongoRelation, STANDALONE_CHECKPOINT_ID } from './MongoRelation.js';
-import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
+import { ChunkedSnapshotQuery } from './MongoSnapshotQuery.js';
+import { CHECKPOINTS_COLLECTION, timestampToDate } from './replication-utils.js';
 /**
  * Thrown when the change stream is not valid anymore, and replication
  * must be restarted.
@@ -31,14 +84,27 @@ export class ChangeStream {
     metrics;
     maxAwaitTimeMS;
     abort_signal;
-    relation_cache = new Map();
+    relationCache = new RelationCache(getCacheIdentifier);
+    /**
+     * Time of the oldest uncommitted change, according to the source db.
+     * This is used to determine the replication lag.
+     */
+    oldestUncommittedChange = null;
+    /**
+     * Keep track of whether we have done a commit or keepalive yet.
+     * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
+     */
+    isStartingReplication = true;
     checkpointStreamId = new mongo.ObjectId();
+    logger;
+    snapshotChunkLength;
     constructor(options) {
         this.storage = options.storage;
         this.metrics = options.metrics;
         this.group_id = options.storage.group_id;
         this.connections = options.connections;
         this.maxAwaitTimeMS = options.maxAwaitTimeMS ?? 10_000;
+        this.snapshotChunkLength = options.snapshotChunkLength ?? 6_000;
         this.client = this.connections.client;
         this.defaultDb = this.connections.db;
         this.sync_rules = options.storage.getParsedSyncRules({
@@ -48,6 +114,7 @@ export class ChangeStream {
         this.abort_signal.addEventListener('abort', () => {
             // TODO: Fast abort?
         }, { once: true });
+        this.logger = options.logger ?? defaultLogger;
     }
     get stopped() {
         return this.abort_signal.aborted;
@@ -58,9 +125,6 @@ export class ChangeStream {
     get configurePostImages() {
         return this.connections.options.postImages == PostImagesOption.AUTO_CONFIGURE;
     }
-    get logPrefix() {
-        return `[powersync_${this.group_id}]`;
-    }
     /**
      * This resolves a pattern, persists the related metadata, and returns
      * the resulting SourceTables.
@@ -88,7 +152,7 @@ export class ChangeStream {
         }, { nameOnly: false })
             .toArray();
         if (!tablePattern.isWildcard && collections.length == 0) {
-            logger.warn(`${this.logPrefix} Collection ${schema}.${tablePattern.name} not found`);
+            this.logger.warn(`Collection ${schema}.${tablePattern.name} not found`);
         }
         for (let collection of collections) {
             const table = await this.handleRelation(batch, getMongoRelation({ db: schema, coll: collection.name }), 
@@ -101,59 +165,162 @@ export class ChangeStream {
     async initSlot() {
         const status = await this.storage.getStatus();
         if (status.snapshot_done && status.checkpoint_lsn) {
-            logger.info(`${this.logPrefix} Initial replication already done`);
-            return { needsInitialSync: false };
+            this.logger.info(`Initial replication already done`);
+            return { needsInitialSync: false, snapshotLsn: null };
         }
-        return { needsInitialSync: true };
+        return { needsInitialSync: true, snapshotLsn: status.snapshot_lsn };
     }
     async estimatedCount(table) {
-        const db = this.client.db(table.schema);
-        const count = await db.collection(table.table).estimatedDocumentCount();
+        const count = await this.estimatedCountNumber(table);
         return `~${count}`;
     }
-    /**
-     * Start initial replication.
-     *
-     * If (partial) replication was done before on this slot, this clears the state
-     * and starts again from scratch.
-     */
-    async startInitialReplication() {
-        await this.storage.clear();
-        await this.initialReplication();
+    async estimatedCountNumber(table) {
+        const db = this.client.db(table.schema);
+        return await db.collection(table.table).estimatedDocumentCount();
     }
-    async initialReplication() {
-        const sourceTables = this.sync_rules.getSourceTables();
-        await this.client.connect();
-        // We need to get the snapshot time before taking the initial snapshot.
-        const hello = await this.defaultDb.command({ hello: 1 });
-        const snapshotTime = hello.lastWrite?.majorityOpTime?.ts;
-        if (hello.msg == 'isdbgrid') {
-            throw new ServiceError(ErrorCode.PSYNC_S1341, 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
+    async getSnapshotLsn() {
+        const env_1 = { stack: [], error: void 0, hasError: false };
+        try {
+            const hello = await this.defaultDb.command({ hello: 1 });
+            // Basic sanity check
+            if (hello.msg == 'isdbgrid') {
+                throw new ServiceError(ErrorCode.PSYNC_S1341, 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
+            }
+            else if (hello.setName == null) {
+                throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
+            }
+            // Open a change stream just to get a resume token for later use.
+            // We could use clusterTime from the hello command, but that won't tell us if the
+            // snapshot isn't valid anymore.
+            // If we just use the first resumeToken from the stream, we get two potential issues:
+            // 1. The resumeToken may just be a wrapped clusterTime, which does not detect changes
+            //    in source db or other stream issues.
+            // 2. The first actual change we get may have the same clusterTime, causing us to incorrect
+            //    skip that event.
+            // Instead, we create a new checkpoint document, and wait until we get that document back in the stream.
+            // To avoid potential race conditions with the checkpoint creation, we create a new checkpoint document
+            // periodically until the timeout is reached.
+            const LSN_TIMEOUT_SECONDS = 60;
+            const LSN_CREATE_INTERVAL_SECONDS = 1;
+            const streamManager = __addDisposableResource(env_1, this.openChangeStream({ lsn: null, maxAwaitTimeMs: 0 }), true);
+            const { stream } = streamManager;
+            const startTime = performance.now();
+            let lastCheckpointCreated = -10_000;
+            let eventsSeen = 0;
+            while (performance.now() - startTime < LSN_TIMEOUT_SECONDS * 1000) {
+                if (performance.now() - lastCheckpointCreated >= LSN_CREATE_INTERVAL_SECONDS * 1000) {
+                    await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                    lastCheckpointCreated = performance.now();
+                }
+                // tryNext() doesn't block, while next() / hasNext() does block until there is data on the stream
+                const changeDocument = await stream.tryNext().catch((e) => {
+                    throw mapChangeStreamError(e);
+                });
+                if (changeDocument == null) {
+                    continue;
+                }
+                const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
+                if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
+                    const checkpointId = changeDocument.documentKey._id;
+                    if (!this.checkpointStreamId.equals(checkpointId)) {
+                        continue;
+                    }
+                    const { comparable: lsn } = new MongoLSN({
+                        timestamp: changeDocument.clusterTime,
+                        resume_token: changeDocument._id
+                    });
+                    return lsn;
+                }
+                eventsSeen += 1;
+            }
+            // Could happen if there is a very large replication lag?
+            throw new ServiceError(ErrorCode.PSYNC_S1301, `Timeout after while waiting for checkpoint document for ${LSN_TIMEOUT_SECONDS}s. Streamed events = ${eventsSeen}`);
         }
-        else if (hello.setName == null) {
-            throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
+        catch (e_1) {
+            env_1.error = e_1;
+            env_1.hasError = true;
         }
-        else if (snapshotTime == null) {
-            // Not known where this would happen apart from the above cases
-            throw new ReplicationAssertionError('MongoDB lastWrite timestamp not found.');
+        finally {
+            const result_1 = __disposeResources(env_1);
+            if (result_1)
+                await result_1;
         }
-        await this.storage.startBatch({ zeroLSN: MongoLSN.ZERO.comparable, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false }, async (batch) => {
+    }
+    async validateSnapshotLsn(lsn) {
+        const env_2 = { stack: [], error: void 0, hasError: false };
+        try {
+            const streamManager = __addDisposableResource(env_2, this.openChangeStream({ lsn: lsn, maxAwaitTimeMs: 0 }), true);
+            const { stream } = streamManager;
+            try {
+                // tryNext() doesn't block, while next() / hasNext() does block until there is data on the stream
+                await stream.tryNext();
+            }
+            catch (e) {
+                // Note: A timeout here is not handled as a ChangeStreamInvalidatedError, even though
+                // we possibly cannot recover from it.
+                throw mapChangeStreamError(e);
+            }
+        }
+        catch (e_2) {
+            env_2.error = e_2;
+            env_2.hasError = true;
+        }
+        finally {
+            const result_2 = __disposeResources(env_2);
+            if (result_2)
+                await result_2;
+        }
+    }
+    async initialReplication(snapshotLsn) {
+        const sourceTables = this.sync_rules.getSourceTables();
+        await this.client.connect();
+        await this.storage.startBatch({
+            logger: this.logger,
+            zeroLSN: MongoLSN.ZERO.comparable,
+            defaultSchema: this.defaultDb.databaseName,
+            storeCurrentData: false,
+            skipExistingRows: true
+        }, async (batch) => {
+            if (snapshotLsn == null) {
+                // First replication attempt - get a snapshot and store the timestamp
+                snapshotLsn = await this.getSnapshotLsn();
+                await batch.setSnapshotLsn(snapshotLsn);
+                this.logger.info(`Marking snapshot at ${snapshotLsn}`);
+            }
+            else {
+                this.logger.info(`Resuming snapshot at ${snapshotLsn}`);
+                // Check that the snapshot is still valid.
+                await this.validateSnapshotLsn(snapshotLsn);
+            }
             // Start by resolving all tables.
             // This checks postImage configuration, and that should fail as
-            // earlier as possible.
+            // early as possible.
             let allSourceTables = [];
             for (let tablePattern of sourceTables) {
                 const tables = await this.resolveQualifiedTableNames(batch, tablePattern);
                 allSourceTables.push(...tables);
             }
+            let tablesWithStatus = [];
             for (let table of allSourceTables) {
+                if (table.snapshotComplete) {
+                    this.logger.info(`Skipping ${table.qualifiedName} - snapshot already done`);
+                    continue;
+                }
+                let count = await this.estimatedCountNumber(table);
+                const updated = await batch.updateTableProgress(table, {
+                    totalEstimatedCount: count
+                });
+                tablesWithStatus.push(updated);
+                this.relationCache.update(updated);
+                this.logger.info(`To replicate: ${table.qualifiedName}: ${updated.snapshotStatus?.replicatedCount}/~${updated.snapshotStatus?.totalEstimatedCount}`);
+            }
+            for (let table of tablesWithStatus) {
                 await this.snapshotTable(batch, table);
                 await batch.markSnapshotDone([table], MongoLSN.ZERO.comparable);
                 await touch();
             }
-            const { comparable: lsn } = new MongoLSN({ timestamp: snapshotTime });
-            logger.info(`${this.logPrefix} Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`);
-            await batch.commit(lsn);
+            this.logger.info(`Snapshot commit at ${snapshotLsn}`);
+            await batch.commit(snapshotLsn);
         });
     }
     async setupCheckpointsCollection() {
@@ -214,51 +381,77 @@ export class ChangeStream {
         }
     }
     async snapshotTable(batch, table) {
-        logger.info(`${this.logPrefix} Replicating ${table.qualifiedName}`);
-        const estimatedCount = await this.estimatedCount(table);
-        let at = 0;
-        const db = this.client.db(table.schema);
-        const collection = db.collection(table.table);
-        const cursor = collection.find({}, { batchSize: 6_000, readConcern: 'majority' });
-        let lastBatch = performance.now();
-        // hasNext() is the call that triggers fetching of the next batch,
-        // then we read it with readBufferedDocuments(). This gives us semi-explicit
-        // control over the fetching of each batch, and avoids a separate promise per document
-        let hasNextPromise = cursor.hasNext();
-        while (await hasNextPromise) {
-            const docBatch = cursor.readBufferedDocuments();
-            // Pre-fetch next batch, so that we can read and write concurrently
-            hasNextPromise = cursor.hasNext();
-            for (let document of docBatch) {
+        const env_3 = { stack: [], error: void 0, hasError: false };
+        try {
+            const totalEstimatedCount = await this.estimatedCountNumber(table);
+            let at = table.snapshotStatus?.replicatedCount ?? 0;
+            const db = this.client.db(table.schema);
+            const collection = db.collection(table.table);
+            const query = __addDisposableResource(env_3, new ChunkedSnapshotQuery({
+                collection,
+                key: table.snapshotStatus?.lastKey,
+                batchSize: this.snapshotChunkLength
+            }), true);
+            if (query.lastKey != null) {
+                this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} - resuming at _id > ${query.lastKey}`);
+            }
+            else {
+                this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()}`);
+            }
+            let lastBatch = performance.now();
+            let nextChunkPromise = query.nextChunk();
+            while (true) {
+                const { docs: docBatch, lastKey } = await nextChunkPromise;
+                if (docBatch.length == 0) {
+                    break;
+                }
                 if (this.abort_signal.aborted) {
                     throw new ReplicationAbortedError(`Aborted initial replication`);
                 }
-                const record = constructAfterRecord(document);
-                // This auto-flushes when the batch reaches its size limit
-                await batch.save({
-                    tag: SaveOperationTag.INSERT,
-                    sourceTable: table,
-                    before: undefined,
-                    beforeReplicaId: undefined,
-                    after: record,
-                    afterReplicaId: document._id
+                // Pre-fetch next batch, so that we can read and write concurrently
+                nextChunkPromise = query.nextChunk();
+                for (let document of docBatch) {
+                    const record = constructAfterRecord(document);
+                    // This auto-flushes when the batch reaches its size limit
+                    await batch.save({
+                        tag: SaveOperationTag.INSERT,
+                        sourceTable: table,
+                        before: undefined,
+                        beforeReplicaId: undefined,
+                        after: record,
+                        afterReplicaId: document._id
+                    });
+                }
+                // Important: flush before marking progress
+                await batch.flush();
+                at += docBatch.length;
+                this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(docBatch.length);
+                table = await batch.updateTableProgress(table, {
+                    lastKey,
+                    replicatedCount: at,
+                    totalEstimatedCount: totalEstimatedCount
                 });
+                this.relationCache.update(table);
+                const duration = performance.now() - lastBatch;
+                lastBatch = performance.now();
+                this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} in ${duration.toFixed(0)}ms`);
+                await touch();
             }
-            at += docBatch.length;
-            this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(docBatch.length);
-            const duration = performance.now() - lastBatch;
-            lastBatch = performance.now();
-            logger.info(`${this.logPrefix} Replicating ${table.qualifiedName} ${at}/${estimatedCount} in ${duration.toFixed(0)}ms`);
-            await touch();
-        }
-        // In case the loop was interrupted, make sure we await the last promise.
-        await hasNextPromise;
-        await batch.flush();
-        logger.info(`${this.logPrefix} Replicated ${at} documents for ${table.qualifiedName}`);
+            // In case the loop was interrupted, make sure we await the last promise.
+            await nextChunkPromise;
+        }
+        catch (e_3) {
+            env_3.error = e_3;
+            env_3.hasError = true;
+        }
+        finally {
+            const result_3 = __disposeResources(env_3);
+            if (result_3)
+                await result_3;
+        }
     }
     async getRelation(batch, descriptor, options) {
-        const cacheId = getCacheIdentifier(descriptor);
-        const existing = this.relation_cache.get(cacheId);
+        const existing = this.relationCache.get(descriptor);
         if (existing != null) {
             return existing;
         }
@@ -287,7 +480,7 @@ export class ChangeStream {
                 collMod: collectionInfo.name,
                 changeStreamPreAndPostImages: { enabled: true }
             });
-            logger.info(`${this.logPrefix} Enabled postImages on ${db}.${collectionInfo.name}`);
+            this.logger.info(`Enabled postImages on ${db}.${collectionInfo.name}`);
         }
         else if (!enabled) {
             throw new ServiceError(ErrorCode.PSYNC_S1343, `postImages not enabled on ${db}.${collectionInfo.name}`);
@@ -309,11 +502,11 @@ export class ChangeStream {
             entity_descriptor: descriptor,
             sync_rules: this.sync_rules
         });
-        this.relation_cache.set(getCacheIdentifier(descriptor), result.table);
+        this.relationCache.update(result.table);
         // Drop conflicting collections.
         // This is generally not expected for MongoDB source dbs, so we log an error.
         if (result.dropTables.length > 0) {
-            logger.error(`Conflicting collections found for ${JSON.stringify(descriptor)}. Dropping: ${result.dropTables.map((t) => t.id).join(', ')}`);
+            this.logger.error(`Conflicting collections found for ${JSON.stringify(descriptor)}. Dropping: ${result.dropTables.map((t) => t.id).join(', ')}`);
             await batch.drop(result.dropTables);
         }
         // Snapshot if:
@@ -322,7 +515,7 @@ export class ChangeStream {
         // 3. The table is used in sync rules.
         const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny;
         if (shouldSnapshot) {
-            logger.info(`${this.logPrefix} New collection: ${descriptor.schema}.${descriptor.name}`);
+            this.logger.info(`New collection: ${descriptor.schema}.${descriptor.name}`);
             // Truncate this table, in case a previous snapshot was interrupted.
             await batch.truncate([result.table]);
             await this.snapshotTable(batch, result.table);
@@ -334,7 +527,7 @@ export class ChangeStream {
     }
     async writeChange(batch, table, change) {
         if (!table.syncAny) {
-            logger.debug(`${this.logPrefix} Collection ${table.qualifiedName} not used in sync rules - skipping`);
+            this.logger.debug(`Collection ${table.qualifiedName} not used in sync rules - skipping`);
             return null;
         }
         this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
@@ -397,7 +590,11 @@ export class ChangeStream {
         const result = await this.initSlot();
         await this.setupCheckpointsCollection();
         if (result.needsInitialSync) {
-            await this.startInitialReplication();
+            if (result.snapshotLsn == null) {
+                // Snapshot LSN is not present, so we need to start replication from scratch.
+                await this.storage.clear({ signal: this.abort_signal });
+            }
+            await this.initialReplication(result.snapshotLsn);
         }
     }
     async streamChanges() {
@@ -413,244 +610,303 @@ export class ChangeStream {
             throw e;
         }
     }
+    openChangeStream(options) {
+        const lastLsn = options.lsn ? MongoLSN.fromSerialized(options.lsn) : null;
+        const startAfter = lastLsn?.timestamp;
+        const resumeAfter = lastLsn?.resumeToken;
+        const filters = this.getSourceNamespaceFilters();
+        const pipeline = [
+            {
+                $match: filters.$match
+            },
+            { $changeStreamSplitLargeEvent: {} }
+        ];
+        let fullDocument;
+        if (this.usePostImages) {
+            // 'read_only' or 'auto_configure'
+            // Configuration happens during snapshot, or when we see new
+            // collections.
+            fullDocument = 'required';
+        }
+        else {
+            fullDocument = 'updateLookup';
+        }
+        const streamOptions = {
+            showExpandedEvents: true,
+            maxAwaitTimeMS: options.maxAwaitTimeMs ?? this.maxAwaitTimeMS,
+            fullDocument: fullDocument
+        };
+        /**
+         * Only one of these options can be supplied at a time.
+         */
+        if (resumeAfter) {
+            streamOptions.resumeAfter = resumeAfter;
+        }
+        else {
+            // Legacy: We don't persist lsns without resumeTokens anymore, but we do still handle the
+            // case if we have an old one.
+            streamOptions.startAtOperationTime = startAfter;
+        }
+        let stream;
+        if (filters.multipleDatabases) {
+            // Requires readAnyDatabase@admin on Atlas
+            stream = this.client.watch(pipeline, streamOptions);
+        }
+        else {
+            // Same general result, but requires less permissions than the above
+            stream = this.defaultDb.watch(pipeline, streamOptions);
+        }
+        this.abort_signal.addEventListener('abort', () => {
+            stream.close();
+        });
+        return {
+            stream,
+            filters,
+            [Symbol.asyncDispose]: async () => {
+                return stream.close();
+            }
+        };
+    }
     async streamChangesInternal() {
         // Auto-activate as soon as initial replication is done
         await this.storage.autoActivate();
-        await this.storage.startBatch({ zeroLSN: MongoLSN.ZERO.comparable, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false }, async (batch) => {
-            const { lastCheckpointLsn } = batch;
-            const lastLsn = lastCheckpointLsn ? MongoLSN.fromSerialized(lastCheckpointLsn) : null;
-            const startAfter = lastLsn?.timestamp;
-            const resumeAfter = lastLsn?.resumeToken;
-            logger.info(`${this.logPrefix} Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`);
-            const filters = this.getSourceNamespaceFilters();
-            const pipeline = [
-                {
-                    $match: filters.$match
-                },
-                { $changeStreamSplitLargeEvent: {} }
-            ];
-            let fullDocument;
-            if (this.usePostImages) {
-                // 'read_only' or 'auto_configure'
-                // Configuration happens during snapshot, or when we see new
-                // collections.
-                fullDocument = 'required';
-            }
-            else {
-                fullDocument = 'updateLookup';
-            }
-            const streamOptions = {
-                showExpandedEvents: true,
-                maxAwaitTimeMS: this.maxAwaitTimeMS,
-                fullDocument: fullDocument
-            };
-            /**
-             * Only one of these options can be supplied at a time.
-             */
-            if (resumeAfter) {
-                streamOptions.resumeAfter = resumeAfter;
-            }
-            else {
-                streamOptions.startAtOperationTime = startAfter;
-            }
-            let stream;
-            if (filters.multipleDatabases) {
-                // Requires readAnyDatabase@admin on Atlas
-                stream = this.client.watch(pipeline, streamOptions);
-            }
-            else {
-                // Same general result, but requires less permissions than the above
-                stream = this.defaultDb.watch(pipeline, streamOptions);
-            }
-            if (this.abort_signal.aborted) {
-                stream.close();
-                return;
-            }
-            this.abort_signal.addEventListener('abort', () => {
-                stream.close();
-            });
-            // Always start with a checkpoint.
-            // This helps us to clear errors when restarting, even if there is
-            // no data to replicate.
-            let waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
-            let splitDocument = null;
-            let flexDbNameWorkaroundLogged = false;
-            let lastEmptyResume = performance.now();
-            while (true) {
+        await this.storage.startBatch({
+            logger: this.logger,
+            zeroLSN: MongoLSN.ZERO.comparable,
+            defaultSchema: this.defaultDb.databaseName,
+            storeCurrentData: false
+        }, async (batch) => {
+            const env_4 = { stack: [], error: void 0, hasError: false };
+            try {
+                const { lastCheckpointLsn } = batch;
+                const lastLsn = MongoLSN.fromSerialized(lastCheckpointLsn);
+                const startAfter = lastLsn?.timestamp;
+                // It is normal for this to be a minute or two old when there is a low volume
+                // of ChangeStream events.
+                const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(startAfter).getTime()) / 1000);
+                this.logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}  | Token age: ${tokenAgeSeconds}s`);
+                const streamManager = __addDisposableResource(env_4, this.openChangeStream({ lsn: lastCheckpointLsn }), true);
+                const { stream, filters } = streamManager;
                 if (this.abort_signal.aborted) {
-                    break;
+                    await stream.close();
+                    return;
                 }
-                const originalChangeDocument = await stream.tryNext().catch((e) => {
-                    throw mapChangeStreamError(e);
-                });
-                // The stream was closed, we will only ever receive `null` from it
-                if (!originalChangeDocument && stream.closed) {
-                    break;
-                }
-                if (this.abort_signal.aborted) {
-                    break;
-                }
-                if (originalChangeDocument == null) {
-                    // We get a new null document after `maxAwaitTimeMS` if there were no other events.
-                    // In this case, stream.resumeToken is the resume token associated with the last response.
-                    // stream.resumeToken is not updated if stream.tryNext() returns data, while stream.next()
-                    // does update it.
-                    // From observed behavior, the actual resumeToken changes around once every 10 seconds.
-                    // If we don't update it on empty events, we do keep consistency, but resuming the stream
-                    // with old tokens may cause connection timeouts.
-                    // We throttle this further by only persisting a keepalive once a minute.
-                    // We add an additional check for waitForCheckpointLsn == null, to make sure we're not
-                    // doing a keepalive in the middle of a transaction.
-                    if (waitForCheckpointLsn == null && performance.now() - lastEmptyResume > 60_000) {
-                        const { comparable: lsn } = MongoLSN.fromResumeToken(stream.resumeToken);
-                        await batch.keepalive(lsn);
-                        await touch();
-                        lastEmptyResume = performance.now();
+                // Always start with a checkpoint.
+                // This helps us to clear errors when restarting, even if there is
+                // no data to replicate.
+                let waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                let splitDocument = null;
+                let flexDbNameWorkaroundLogged = false;
+                let lastEmptyResume = performance.now();
+                while (true) {
+                    if (this.abort_signal.aborted) {
+                        break;
                     }
-                    continue;
-                }
-                await touch();
-                if (startAfter != null && originalChangeDocument.clusterTime?.lte(startAfter)) {
-                    continue;
-                }
-                let changeDocument = originalChangeDocument;
-                if (originalChangeDocument?.splitEvent != null) {
-                    // Handle split events from $changeStreamSplitLargeEvent.
-                    // This is only relevant for very large update operations.
-                    const splitEvent = originalChangeDocument?.splitEvent;
-                    if (splitDocument == null) {
-                        splitDocument = originalChangeDocument;
-                    }
-                    else {
-                        splitDocument = Object.assign(splitDocument, originalChangeDocument);
+                    const originalChangeDocument = await stream.tryNext().catch((e) => {
+                        throw mapChangeStreamError(e);
+                    });
+                    // The stream was closed, we will only ever receive `null` from it
+                    if (!originalChangeDocument && stream.closed) {
+                        break;
                     }
-                    if (splitEvent.fragment == splitEvent.of) {
-                        // Got all fragments
-                        changeDocument = splitDocument;
-                        splitDocument = null;
+                    if (this.abort_signal.aborted) {
+                        break;
                     }
-                    else {
-                        // Wait for more fragments
+                    if (originalChangeDocument == null) {
+                        // We get a new null document after `maxAwaitTimeMS` if there were no other events.
+                        // In this case, stream.resumeToken is the resume token associated with the last response.
+                        // stream.resumeToken is not updated if stream.tryNext() returns data, while stream.next()
+                        // does update it.
+                        // From observed behavior, the actual resumeToken changes around once every 10 seconds.
+                        // If we don't update it on empty events, we do keep consistency, but resuming the stream
+                        // with old tokens may cause connection timeouts.
+                        // We throttle this further by only persisting a keepalive once a minute.
+                        // We add an additional check for waitForCheckpointLsn == null, to make sure we're not
+                        // doing a keepalive in the middle of a transaction.
+                        if (waitForCheckpointLsn == null && performance.now() - lastEmptyResume > 60_000) {
+                            const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(stream.resumeToken);
+                            await batch.keepalive(lsn);
+                            await touch();
+                            lastEmptyResume = performance.now();
+                            // Log the token update. This helps as a general "replication is still active" message in the logs.
+                            // This token would typically be around 10s behind.
+                            this.logger.info(`Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`);
+                            this.isStartingReplication = false;
+                        }
                         continue;
                     }
-                }
-                else if (splitDocument != null) {
-                    // We were waiting for fragments, but got a different event
-                    throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
-                }
-                if (!filters.multipleDatabases &&
-                    'ns' in changeDocument &&
-                    changeDocument.ns.db != this.defaultDb.databaseName &&
-                    changeDocument.ns.db.endsWith(`_${this.defaultDb.databaseName}`)) {
-                    // When all of the following conditions are met:
-                    // 1. We're replicating from an Atlas Flex instance.
-                    // 2. There were changestream events recorded while the PowerSync service is paused.
-                    // 3. We're only replicating from a single database.
-                    // Then we've observed an ns with for example {db: '67b83e86cd20730f1e766dde_ps'},
-                    // instead of the expected {db: 'ps'}.
-                    // We correct this.
-                    changeDocument.ns.db = this.defaultDb.databaseName;
-                    if (!flexDbNameWorkaroundLogged) {
-                        flexDbNameWorkaroundLogged = true;
-                        logger.warn(`${this.logPrefix} Incorrect DB name in change stream: ${changeDocument.ns.db}. Changed to ${this.defaultDb.databaseName}.`);
-                    }
-                }
-                const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
-                if (ns?.coll == CHECKPOINTS_COLLECTION) {
-                    /**
-                     * Dropping the database does not provide an `invalidate` event.
-                     * We typically would receive `drop` events for the collection which we
-                     * would process below.
-                     *
-                     * However we don't commit the LSN after collections are dropped.
-                     * The prevents the `startAfter` or `resumeToken` from advancing past the drop events.
-                     * The stream also closes after the drop events.
-                     * This causes an infinite loop of processing the collection drop events.
-                     *
-                     * This check here invalidates the change stream if our `_checkpoints` collection
-                     * is dropped. This allows for detecting when the DB is dropped.
-                     */
-                    if (changeDocument.operationType == 'drop') {
-                        throw new ChangeStreamInvalidatedError('Internal collections have been dropped', new Error('_checkpoints collection was dropped'));
-                    }
-                    if (!(changeDocument.operationType == 'insert' ||
-                        changeDocument.operationType == 'update' ||
-                        changeDocument.operationType == 'replace')) {
+                    await touch();
+                    if (startAfter != null && originalChangeDocument.clusterTime?.lte(startAfter)) {
                         continue;
                     }
-                    // We handle two types of checkpoint events:
-                    // 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
-                    //    immediately, regardless of where they were created.
-                    // 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
-                    //    limiting of commits, so we specifically want to exclude checkpoints from other streams.
-                    //
-                    // It may be useful to also throttle commits due to standalone checkpoints in the future.
-                    // However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
-                    const checkpointId = changeDocument.documentKey._id;
-                    if (!(checkpointId == STANDALONE_CHECKPOINT_ID || this.checkpointStreamId.equals(checkpointId))) {
-                        continue;
+                    let changeDocument = originalChangeDocument;
+                    if (originalChangeDocument?.splitEvent != null) {
+                        // Handle split events from $changeStreamSplitLargeEvent.
+                        // This is only relevant for very large update operations.
+                        const splitEvent = originalChangeDocument?.splitEvent;
+                        if (splitDocument == null) {
+                            splitDocument = originalChangeDocument;
+                        }
+                        else {
+                            splitDocument = Object.assign(splitDocument, originalChangeDocument);
+                        }
+                        if (splitEvent.fragment == splitEvent.of) {
+                            // Got all fragments
+                            changeDocument = splitDocument;
+                            splitDocument = null;
+                        }
+                        else {
+                            // Wait for more fragments
+                            continue;
+                        }
                     }
-                    const { comparable: lsn } = new MongoLSN({
-                        timestamp: changeDocument.clusterTime,
-                        resume_token: changeDocument._id
-                    });
-                    if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
-                        waitForCheckpointLsn = null;
+                    else if (splitDocument != null) {
+                        // We were waiting for fragments, but got a different event
+                        throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
                     }
-                    await batch.commit(lsn);
-                }
-                else if (changeDocument.operationType == 'insert' ||
-                    changeDocument.operationType == 'update' ||
-                    changeDocument.operationType == 'replace' ||
-                    changeDocument.operationType == 'delete') {
-                    if (waitForCheckpointLsn == null) {
-                        waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                    if (!filters.multipleDatabases &&
+                        'ns' in changeDocument &&
+                        changeDocument.ns.db != this.defaultDb.databaseName &&
+                        changeDocument.ns.db.endsWith(`_${this.defaultDb.databaseName}`)) {
+                        // When all of the following conditions are met:
+                        // 1. We're replicating from an Atlas Flex instance.
+                        // 2. There were changestream events recorded while the PowerSync service is paused.
+                        // 3. We're only replicating from a single database.
+                        // Then we've observed an ns with for example {db: '67b83e86cd20730f1e766dde_ps'},
+                        // instead of the expected {db: 'ps'}.
+                        // We correct this.
+                        changeDocument.ns.db = this.defaultDb.databaseName;
+                        if (!flexDbNameWorkaroundLogged) {
+                            flexDbNameWorkaroundLogged = true;
+                            this.logger.warn(`Incorrect DB name in change stream: ${changeDocument.ns.db}. Changed to ${this.defaultDb.databaseName}.`);
+                        }
                     }
-                    const rel = getMongoRelation(changeDocument.ns);
-                    const table = await this.getRelation(batch, rel, {
-                        // In most cases, we should not need to snapshot this. But if this is the first time we see the collection
-                        // for whatever reason, then we do need to snapshot it.
-                        // This may result in some duplicate operations when a collection is created for the first time after
-                        // sync rules was deployed.
-                        snapshot: true
-                    });
-                    if (table.syncAny) {
-                        await this.writeChange(batch, table, changeDocument);
+                    const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
+                    if (ns?.coll == CHECKPOINTS_COLLECTION) {
+                        /**
+                         * Dropping the database does not provide an `invalidate` event.
+                         * We typically would receive `drop` events for the collection which we
+                         * would process below.
+                         *
+                         * However we don't commit the LSN after collections are dropped.
+                         * The prevents the `startAfter` or `resumeToken` from advancing past the drop events.
+                         * The stream also closes after the drop events.
+                         * This causes an infinite loop of processing the collection drop events.
+                         *
+                         * This check here invalidates the change stream if our `_checkpoints` collection
+                         * is dropped. This allows for detecting when the DB is dropped.
+                         */
+                        if (changeDocument.operationType == 'drop') {
+                            throw new ChangeStreamInvalidatedError('Internal collections have been dropped', new Error('_checkpoints collection was dropped'));
+                        }
+                        if (!(changeDocument.operationType == 'insert' ||
+                            changeDocument.operationType == 'update' ||
+                            changeDocument.operationType == 'replace')) {
+                            continue;
+                        }
+                        // We handle two types of checkpoint events:
+                        // 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
+                        //    immediately, regardless of where they were created.
+                        // 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
+                        //    limiting of commits, so we specifically want to exclude checkpoints from other streams.
+                        //
+                        // It may be useful to also throttle commits due to standalone checkpoints in the future.
+                        // However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
+                        const checkpointId = changeDocument.documentKey._id;
+                        if (!(checkpointId == STANDALONE_CHECKPOINT_ID || this.checkpointStreamId.equals(checkpointId))) {
+                            continue;
+                        }
+                        const { comparable: lsn } = new MongoLSN({
+                            timestamp: changeDocument.clusterTime,
+                            resume_token: changeDocument._id
+                        });
+                        if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
+                            waitForCheckpointLsn = null;
+                        }
+                        const didCommit = await batch.commit(lsn, { oldestUncommittedChange: this.oldestUncommittedChange });
+                        if (didCommit) {
+                            this.oldestUncommittedChange = null;
+                            this.isStartingReplication = false;
+                        }
                     }
-                }
-                else if (changeDocument.operationType == 'drop') {
-                    const rel = getMongoRelation(changeDocument.ns);
-                    const table = await this.getRelation(batch, rel, {
-                        // We're "dropping" this collection, so never snapshot it.
-                        snapshot: false
-                    });
-                    if (table.syncAny) {
-                        await batch.drop([table]);
-                        this.relation_cache.delete(getCacheIdentifier(rel));
+                    else if (changeDocument.operationType == 'insert' ||
+                        changeDocument.operationType == 'update' ||
+                        changeDocument.operationType == 'replace' ||
+                        changeDocument.operationType == 'delete') {
+                        if (waitForCheckpointLsn == null) {
+                            waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                        }
+                        const rel = getMongoRelation(changeDocument.ns);
+                        const table = await this.getRelation(batch, rel, {
+                            // In most cases, we should not need to snapshot this. But if this is the first time we see the collection
+                            // for whatever reason, then we do need to snapshot it.
+                            // This may result in some duplicate operations when a collection is created for the first time after
+                            // sync rules was deployed.
+                            snapshot: true
+                        });
+                        if (table.syncAny) {
+                            if (this.oldestUncommittedChange == null && changeDocument.clusterTime != null) {
+                                this.oldestUncommittedChange = timestampToDate(changeDocument.clusterTime);
+                            }
+                            await this.writeChange(batch, table, changeDocument);
+                        }
                     }
-                }
-                else if (changeDocument.operationType == 'rename') {
-                    const relFrom = getMongoRelation(changeDocument.ns);
-                    const relTo = getMongoRelation(changeDocument.to);
-                    const tableFrom = await this.getRelation(batch, relFrom, {
-                        // We're "dropping" this collection, so never snapshot it.
-                        snapshot: false
-                    });
-                    if (tableFrom.syncAny) {
-                        await batch.drop([tableFrom]);
-                        this.relation_cache.delete(getCacheIdentifier(relFrom));
+                    else if (changeDocument.operationType == 'drop') {
+                        const rel = getMongoRelation(changeDocument.ns);
+                        const table = await this.getRelation(batch, rel, {
+                            // We're "dropping" this collection, so never snapshot it.
+                            snapshot: false
+                        });
+                        if (table.syncAny) {
+                            await batch.drop([table]);
+                            this.relationCache.delete(table);
+                        }
+                    }
+                    else if (changeDocument.operationType == 'rename') {
+                        const relFrom = getMongoRelation(changeDocument.ns);
+                        const relTo = getMongoRelation(changeDocument.to);
+                        const tableFrom = await this.getRelation(batch, relFrom, {
+                            // We're "dropping" this collection, so never snapshot it.
+                            snapshot: false
+                        });
+                        if (tableFrom.syncAny) {
+                            await batch.drop([tableFrom]);
+                            this.relationCache.delete(relFrom);
+                        }
+                        // Here we do need to snapshot the new table
+                        const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
+                        await this.handleRelation(batch, relTo, {
+                            // This is a new (renamed) collection, so always snapshot it.
+                            snapshot: true,
+                            collectionInfo: collection
+                        });
                     }
-                    // Here we do need to snapshot the new table
-                    const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
-                    await this.handleRelation(batch, relTo, {
-                        // This is a new (renamed) collection, so always snapshot it.
-                        snapshot: true,
-                        collectionInfo: collection
-                    });
                 }
             }
+            catch (e_4) {
+                env_4.error = e_4;
+                env_4.hasError = true;
+            }
+            finally {
+                const result_4 = __disposeResources(env_4);
+                if (result_4)
+                    await result_4;
+            }
         });
     }
+    async getReplicationLagMillis() {
+        if (this.oldestUncommittedChange == null) {
+            if (this.isStartingReplication) {
+                // We don't have anything to compute replication lag with yet.
+                return undefined;
+            }
+            else {
+                // We don't have any uncommitted changes, so replication is up-to-date.
+                return 0;
+            }
+        }
+        return Date.now() - this.oldestUncommittedChange.getTime();
+    }
 }
 async function touch() {
     // FIXME: The hosted Kubernetes probe does not actually check the timestamp on this.