@powersync/service-module-mongodb 0.15.1 → 0.15.3

package/dist/replication/ChangeStream.js

@@ -1,62 +1,11 @@
-var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
-    if (value !== null && value !== void 0) {
-        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
-        var dispose, inner;
-        if (async) {
-            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
-            dispose = value[Symbol.asyncDispose];
-        }
-        if (dispose === void 0) {
-            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
-            dispose = value[Symbol.dispose];
-            if (async) inner = dispose;
-        }
-        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
-        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
-        env.stack.push({ value: value, dispose: dispose, async: async });
-    }
-    else if (async) {
-        env.stack.push({ async: true });
-    }
-    return value;
-};
-var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
-    return function (env) {
-        function fail(e) {
-            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
-            env.hasError = true;
-        }
-        var r, s = 0;
-        function next() {
-            while (r = env.stack.pop()) {
-                try {
-                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
-                    if (r.dispose) {
-                        var result = r.dispose.call(r.value);
-                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
-                    }
-                    else s |= 1;
-                }
-                catch (e) {
-                    fail(e);
-                }
-            }
-            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
-            if (env.hasError) throw env.error;
-        }
-        return next();
-    };
-})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
-    var e = new Error(message);
-    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
-});
 import { isMongoNetworkTimeoutError, isMongoServerError, mongo } from '@powersync/lib-service-mongodb';
 import { container, DatabaseConnectionError, logger as defaultLogger, ErrorCode, ReplicationAbortedError, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
-import { RelationCache, SaveOperationTag } from '@powersync/service-core';
+import { RelationCache, ReplicationLagTracker, SaveOperationTag } from '@powersync/service-core';
 import { ReplicationMetric } from '@powersync/service-types';
 import { MongoLSN } from '../common/MongoLSN.js';
 import { PostImagesOption } from '../types/types.js';
 import { escapeRegExp } from '../utils.js';
+import { trackChangeStreamBsonBytes } from './internal-mongodb-utils.js';
 import { constructAfterRecord, createCheckpoint, getCacheIdentifier, getMongoRelation, STANDALONE_CHECKPOINT_ID } from './MongoRelation.js';
 import { ChunkedSnapshotQuery } from './MongoSnapshotQuery.js';
 import { CHECKPOINTS_COLLECTION, timestampToDate } from './replication-utils.js';
@@ -85,16 +34,7 @@ export class ChangeStream {
     maxAwaitTimeMS;
     abort_signal;
     relationCache = new RelationCache(getCacheIdentifier);
-    /**
-     * Time of the oldest uncommitted change, according to the source db.
-     * This is used to determine the replication lag.
-     */
-    oldestUncommittedChange = null;
-    /**
-     * Keep track of whether we have done a commit or keepalive yet.
-     * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
-     */
-    isStartingReplication = true;
+    replicationLag = new ReplicationLagTracker();
     checkpointStreamId = new mongo.ObjectId();
     logger;
     snapshotChunkLength;
@@ -188,101 +128,77 @@ export class ChangeStream {
      * This LSN can survive initial replication restarts.
      */
     async getSnapshotLsn() {
-        const env_1 = { stack: [], error: void 0, hasError: false };
-        try {
-            const hello = await this.defaultDb.command({ hello: 1 });
-            // Basic sanity check
-            if (hello.msg == 'isdbgrid') {
-                throw new ServiceError(ErrorCode.PSYNC_S1341, 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
+        const hello = await this.defaultDb.command({ hello: 1 });
+        // Basic sanity check
+        if (hello.msg == 'isdbgrid') {
+            throw new ServiceError(ErrorCode.PSYNC_S1341, 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
+        }
+        else if (hello.setName == null) {
+            throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
+        }
+        // Open a change stream just to get a resume token for later use.
+        // We could use clusterTime from the hello command, but that won't tell us if the
+        // snapshot isn't valid anymore.
+        // If we just use the first resumeToken from the stream, we get two potential issues:
+        // 1. The resumeToken may just be a wrapped clusterTime, which does not detect changes
+        //    in source db or other stream issues.
+        // 2. The first actual change we get may have the same clusterTime, causing us to incorrect
+        //    skip that event.
+        // Instead, we create a new checkpoint document, and wait until we get that document back in the stream.
+        // To avoid potential race conditions with the checkpoint creation, we create a new checkpoint document
+        // periodically until the timeout is reached.
+        const LSN_TIMEOUT_SECONDS = 60;
+        const LSN_CREATE_INTERVAL_SECONDS = 1;
+        // Create a checkpoint, and open a change stream using startAtOperationTime with the checkpoint's operationTime.
+        const firstCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+        await using streamManager = this.openChangeStream({ lsn: firstCheckpointLsn, maxAwaitTimeMs: 0 });
+        const { stream } = streamManager;
+        const startTime = performance.now();
+        let lastCheckpointCreated = performance.now();
+        let eventsSeen = 0;
+        while (performance.now() - startTime < LSN_TIMEOUT_SECONDS * 1000) {
+            if (performance.now() - lastCheckpointCreated >= LSN_CREATE_INTERVAL_SECONDS * 1000) {
+                await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                lastCheckpointCreated = performance.now();
             }
-            else if (hello.setName == null) {
-                throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
+            // tryNext() doesn't block, while next() / hasNext() does block until there is data on the stream
+            const changeDocument = await stream.tryNext().catch((e) => {
+                throw mapChangeStreamError(e);
+            });
+            if (changeDocument == null) {
+                continue;
             }
-            // Open a change stream just to get a resume token for later use.
-            // We could use clusterTime from the hello command, but that won't tell us if the
-            // snapshot isn't valid anymore.
-            // If we just use the first resumeToken from the stream, we get two potential issues:
-            // 1. The resumeToken may just be a wrapped clusterTime, which does not detect changes
-            //    in source db or other stream issues.
-            // 2. The first actual change we get may have the same clusterTime, causing us to incorrect
-            //    skip that event.
-            // Instead, we create a new checkpoint document, and wait until we get that document back in the stream.
-            // To avoid potential race conditions with the checkpoint creation, we create a new checkpoint document
-            // periodically until the timeout is reached.
-            const LSN_TIMEOUT_SECONDS = 60;
-            const LSN_CREATE_INTERVAL_SECONDS = 1;
-            // Create a checkpoint, and open a change stream using startAtOperationTime with the checkpoint's operationTime.
-            const firstCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
-            const streamManager = __addDisposableResource(env_1, this.openChangeStream({ lsn: firstCheckpointLsn, maxAwaitTimeMs: 0 }), true);
-            const { stream } = streamManager;
-            const startTime = performance.now();
-            let lastCheckpointCreated = performance.now();
-            let eventsSeen = 0;
-            while (performance.now() - startTime < LSN_TIMEOUT_SECONDS * 1000) {
-                if (performance.now() - lastCheckpointCreated >= LSN_CREATE_INTERVAL_SECONDS * 1000) {
-                    await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
-                    lastCheckpointCreated = performance.now();
-                }
-                // tryNext() doesn't block, while next() / hasNext() does block until there is data on the stream
-                const changeDocument = await stream.tryNext().catch((e) => {
-                    throw mapChangeStreamError(e);
-                });
-                if (changeDocument == null) {
+            const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
+            if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
+                const checkpointId = changeDocument.documentKey._id;
+                if (!this.checkpointStreamId.equals(checkpointId)) {
                     continue;
                 }
-                const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
-                if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
-                    const checkpointId = changeDocument.documentKey._id;
-                    if (!this.checkpointStreamId.equals(checkpointId)) {
-                        continue;
-                    }
-                    const { comparable: lsn } = new MongoLSN({
-                        timestamp: changeDocument.clusterTime,
-                        resume_token: changeDocument._id
-                    });
-                    return lsn;
-                }
-                eventsSeen += 1;
+                const { comparable: lsn } = new MongoLSN({
+                    timestamp: changeDocument.clusterTime,
+                    resume_token: changeDocument._id
+                });
+                return lsn;
             }
-            // Could happen if there is a very large replication lag?
-            throw new ServiceError(ErrorCode.PSYNC_S1301, `Timeout after while waiting for checkpoint document for ${LSN_TIMEOUT_SECONDS}s. Streamed events = ${eventsSeen}`);
-        }
-        catch (e_1) {
-            env_1.error = e_1;
-            env_1.hasError = true;
-        }
-        finally {
-            const result_1 = __disposeResources(env_1);
-            if (result_1)
-                await result_1;
+            eventsSeen += 1;
         }
+        // Could happen if there is a very large replication lag?
+        throw new ServiceError(ErrorCode.PSYNC_S1301, `Timeout after while waiting for checkpoint document for ${LSN_TIMEOUT_SECONDS}s. Streamed events = ${eventsSeen}`);
     }
     /**
      * Given a snapshot LSN, validate that we can read from it, by opening a change stream.
      */
     async validateSnapshotLsn(lsn) {
-        const env_2 = { stack: [], error: void 0, hasError: false };
+        await using streamManager = this.openChangeStream({ lsn: lsn, maxAwaitTimeMs: 0 });
+        const { stream } = streamManager;
         try {
-            const streamManager = __addDisposableResource(env_2, this.openChangeStream({ lsn: lsn, maxAwaitTimeMs: 0 }), true);
-            const { stream } = streamManager;
-            try {
-                // tryNext() doesn't block, while next() / hasNext() does block until there is data on the stream
-                await stream.tryNext();
-            }
-            catch (e) {
-                // Note: A timeout here is not handled as a ChangeStreamInvalidatedError, even though
-                // we possibly cannot recover from it.
-                throw mapChangeStreamError(e);
-            }
+            // tryNext() doesn't block, while next() / hasNext() does block until there is data on the stream
+            await stream.tryNext();
         }
-        catch (e_2) {
-            env_2.error = e_2;
-            env_2.hasError = true;
-        }
-        finally {
-            const result_2 = __disposeResources(env_2);
-            if (result_2)
-                await result_2;
+        catch (e) {
+            // Note: A timeout here is not handled as a ChangeStreamInvalidatedError, even though
+            // we possibly cannot recover from it.
+            throw mapChangeStreamError(e);
         }
     }
     async initialReplication(snapshotLsn) {
@@ -420,75 +336,68 @@ export class ChangeStream {
         }
     }
     async snapshotTable(batch, table) {
-        const env_3 = { stack: [], error: void 0, hasError: false };
-        try {
-            const totalEstimatedCount = await this.estimatedCountNumber(table);
-            let at = table.snapshotStatus?.replicatedCount ?? 0;
-            const db = this.client.db(table.schema);
-            const collection = db.collection(table.name);
-            const query = __addDisposableResource(env_3, new ChunkedSnapshotQuery({
-                collection,
-                key: table.snapshotStatus?.lastKey,
-                batchSize: this.snapshotChunkLength
-            }), true);
-            if (query.lastKey != null) {
-                this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} - resuming at _id > ${query.lastKey}`);
+        const rowsReplicatedMetric = this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED);
+        const bytesReplicatedMetric = this.metrics.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES);
+        const chunksReplicatedMetric = this.metrics.getCounter(ReplicationMetric.CHUNKS_REPLICATED);
+        const totalEstimatedCount = await this.estimatedCountNumber(table);
+        let at = table.snapshotStatus?.replicatedCount ?? 0;
+        const db = this.client.db(table.schema);
+        const collection = db.collection(table.name);
+        await using query = new ChunkedSnapshotQuery({
+            collection,
+            key: table.snapshotStatus?.lastKey,
+            batchSize: this.snapshotChunkLength
+        });
+        if (query.lastKey != null) {
+            this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} - resuming at _id > ${query.lastKey}`);
+        }
+        else {
+            this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()}`);
+        }
+        let lastBatch = performance.now();
+        let nextChunkPromise = query.nextChunk();
+        while (true) {
+            const { docs: docBatch, lastKey, bytes: chunkBytes } = await nextChunkPromise;
+            if (docBatch.length == 0) {
+                // No more data - stop iterating
+                break;
             }
-            else {
-                this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()}`);
+            bytesReplicatedMetric.add(chunkBytes);
+            chunksReplicatedMetric.add(1);
+            if (this.abort_signal.aborted) {
+                throw new ReplicationAbortedError(`Aborted initial replication`, this.abort_signal.reason);
             }
-            let lastBatch = performance.now();
-            let nextChunkPromise = query.nextChunk();
-            while (true) {
-                const { docs: docBatch, lastKey } = await nextChunkPromise;
-                if (docBatch.length == 0) {
-                    // No more data - stop iterating
-                    break;
-                }
-                if (this.abort_signal.aborted) {
-                    throw new ReplicationAbortedError(`Aborted initial replication`, this.abort_signal.reason);
-                }
-                // Pre-fetch next batch, so that we can read and write concurrently
-                nextChunkPromise = query.nextChunk();
-                for (let document of docBatch) {
-                    const record = this.constructAfterRecord(document);
-                    // This auto-flushes when the batch reaches its size limit
-                    await batch.save({
-                        tag: SaveOperationTag.INSERT,
-                        sourceTable: table,
-                        before: undefined,
-                        beforeReplicaId: undefined,
-                        after: record,
-                        afterReplicaId: document._id
-                    });
-                }
-                // Important: flush before marking progress
-                await batch.flush();
-                at += docBatch.length;
-                this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(docBatch.length);
-                table = await batch.updateTableProgress(table, {
-                    lastKey,
-                    replicatedCount: at,
-                    totalEstimatedCount: totalEstimatedCount
+            // Pre-fetch next batch, so that we can read and write concurrently
+            nextChunkPromise = query.nextChunk();
+            for (let document of docBatch) {
+                const record = this.constructAfterRecord(document);
+                // This auto-flushes when the batch reaches its size limit
+                await batch.save({
+                    tag: SaveOperationTag.INSERT,
+                    sourceTable: table,
+                    before: undefined,
+                    beforeReplicaId: undefined,
+                    after: record,
+                    afterReplicaId: document._id
                 });
-                this.relationCache.update(table);
-                const duration = performance.now() - lastBatch;
-                lastBatch = performance.now();
-                this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} in ${duration.toFixed(0)}ms`);
-                this.touch();
             }
-            // In case the loop was interrupted, make sure we await the last promise.
-            await nextChunkPromise;
-        }
-        catch (e_3) {
-            env_3.error = e_3;
-            env_3.hasError = true;
-        }
-        finally {
-            const result_3 = __disposeResources(env_3);
-            if (result_3)
-                await result_3;
+            // Important: flush before marking progress
+            await batch.flush();
+            at += docBatch.length;
+            rowsReplicatedMetric.add(docBatch.length);
+            table = await batch.updateTableProgress(table, {
+                lastKey,
+                replicatedCount: at,
+                totalEstimatedCount: totalEstimatedCount
+            });
+            this.relationCache.update(table);
+            const duration = performance.now() - lastBatch;
+            lastBatch = performance.now();
+            this.logger.info(`Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} in ${duration.toFixed(0)}ms`);
+            this.touch();
         }
+        // In case the loop was interrupted, make sure we await the last promise.
+        await nextChunkPromise;
     }
     async getRelation(batch, descriptor, options) {
         const existing = this.relationCache.get(descriptor);
@@ -721,7 +630,22 @@ export class ChangeStream {
             }
         };
     }
+    getBufferedChangeCount(stream) {
+        // The driver keeps fetched change stream documents on the underlying cursor, but does
+        // not expose that through the public ChangeStream API. We use this to detect backlog
+        // building up before we have processed the corresponding source changes locally.
+        // If the driver API changes, we'll have a hard error here.
+        // We specifically want to avoid a silent performance regression if the driver behavior changes.
+        const cursor = stream.cursor;
+        if (cursor == null || typeof cursor.bufferedCount != 'function') {
+            throw new ReplicationAssertionError('MongoDB ChangeStream no longer exposes an internal cursor with bufferedCount');
+        }
+        return cursor.bufferedCount();
+    }
     async streamChangesInternal() {
+        const transactionsReplicatedMetric = this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED);
+        const bytesReplicatedMetric = this.metrics.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES);
+        const chunksReplicatedMetric = this.metrics.getCounter(ReplicationMetric.CHUNKS_REPLICATED);
         await this.storage.startBatch({
             logger: this.logger,
             zeroLSN: MongoLSN.ZERO.comparable,
@@ -729,265 +653,267 @@ export class ChangeStream {
             // We get a complete postimage for every change, so we don't need to store the current data.
             storeCurrentData: false
         }, async (batch) => {
-            const env_4 = { stack: [], error: void 0, hasError: false };
-            try {
-                const { resumeFromLsn } = batch;
-                if (resumeFromLsn == null) {
-                    throw new ReplicationAssertionError(`No LSN found to resume from`);
+            const { resumeFromLsn } = batch;
+            if (resumeFromLsn == null) {
+                throw new ReplicationAssertionError(`No LSN found to resume from`);
+            }
+            const lastLsn = MongoLSN.fromSerialized(resumeFromLsn);
+            const startAfter = lastLsn?.timestamp;
+            // It is normal for this to be a minute or two old when there is a low volume
+            // of ChangeStream events.
+            const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(startAfter).getTime()) / 1000);
+            this.logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}  | Token age: ${tokenAgeSeconds}s`);
+            await using streamManager = this.openChangeStream({ lsn: resumeFromLsn });
+            const { stream, filters } = streamManager;
+            if (this.abort_signal.aborted) {
+                await stream.close();
+                return;
+            }
+            trackChangeStreamBsonBytes(stream, (bytes) => {
+                bytesReplicatedMetric.add(bytes);
+                // Each of these represent a single response message from MongoDB.
+                chunksReplicatedMetric.add(1);
+            });
+            // Always start with a checkpoint.
+            // This helps us to clear errors when restarting, even if there is
+            // no data to replicate.
+            let waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+            let splitDocument = null;
+            let flexDbNameWorkaroundLogged = false;
+            let changesSinceLastCheckpoint = 0;
+            let lastEmptyResume = performance.now();
+            let lastTxnKey = null;
+            while (true) {
+                if (this.abort_signal.aborted) {
+                    break;
+                }
+                const originalChangeDocument = await stream.tryNext().catch((e) => {
+                    throw mapChangeStreamError(e);
+                });
+                // The stream was closed, we will only ever receive `null` from it
+                if (!originalChangeDocument && stream.closed) {
+                    break;
                 }
-                const lastLsn = MongoLSN.fromSerialized(resumeFromLsn);
-                const startAfter = lastLsn?.timestamp;
-                // It is normal for this to be a minute or two old when there is a low volume
-                // of ChangeStream events.
-                const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(startAfter).getTime()) / 1000);
-                this.logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}  | Token age: ${tokenAgeSeconds}s`);
-                const streamManager = __addDisposableResource(env_4, this.openChangeStream({ lsn: resumeFromLsn }), true);
-                const { stream, filters } = streamManager;
                 if (this.abort_signal.aborted) {
-                    await stream.close();
-                    return;
+                    break;
                 }
-                // Always start with a checkpoint.
-                // This helps us to clear errors when restarting, even if there is
-                // no data to replicate.
-                let waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
-                let splitDocument = null;
-                let flexDbNameWorkaroundLogged = false;
-                let changesSinceLastCheckpoint = 0;
-                let lastEmptyResume = performance.now();
-                while (true) {
-                    if (this.abort_signal.aborted) {
-                        break;
+                if (originalChangeDocument == null) {
+                    // We get a new null document after `maxAwaitTimeMS` if there were no other events.
+                    // In this case, stream.resumeToken is the resume token associated with the last response.
+                    // stream.resumeToken is not updated if stream.tryNext() returns data, while stream.next()
+                    // does update it.
+                    // From observed behavior, the actual resumeToken changes around once every 10 seconds.
+                    // If we don't update it on empty events, we do keep consistency, but resuming the stream
+                    // with old tokens may cause connection timeouts.
+                    // We throttle this further by only persisting a keepalive once a minute.
+                    // We add an additional check for waitForCheckpointLsn == null, to make sure we're not
+                    // doing a keepalive in the middle of a transaction.
+                    if (waitForCheckpointLsn == null && performance.now() - lastEmptyResume > 60_000) {
+                        const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(stream.resumeToken);
+                        await batch.keepalive(lsn);
+                        this.touch();
+                        lastEmptyResume = performance.now();
+                        // Log the token update. This helps as a general "replication is still active" message in the logs.
+                        // This token would typically be around 10s behind.
+                        this.logger.info(`Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`);
+                        this.replicationLag.markStarted();
                     }
-                    const originalChangeDocument = await stream.tryNext().catch((e) => {
-                        throw mapChangeStreamError(e);
-                    });
-                    // The stream was closed, we will only ever receive `null` from it
-                    if (!originalChangeDocument && stream.closed) {
-                        break;
+                    continue;
+                }
+                this.touch();
+                if (startAfter != null && originalChangeDocument.clusterTime?.lte(startAfter)) {
+                    continue;
+                }
+                let changeDocument = originalChangeDocument;
+                if (originalChangeDocument?.splitEvent != null) {
+                    // Handle split events from $changeStreamSplitLargeEvent.
+                    // This is only relevant for very large update operations.
+                    const splitEvent = originalChangeDocument?.splitEvent;
+                    if (splitDocument == null) {
+                        splitDocument = originalChangeDocument;
                     }
-                    if (this.abort_signal.aborted) {
-                        break;
+                    else {
+                        splitDocument = Object.assign(splitDocument, originalChangeDocument);
                     }
-                    if (originalChangeDocument == null) {
-                        // We get a new null document after `maxAwaitTimeMS` if there were no other events.
-                        // In this case, stream.resumeToken is the resume token associated with the last response.
-                        // stream.resumeToken is not updated if stream.tryNext() returns data, while stream.next()
-                        // does update it.
-                        // From observed behavior, the actual resumeToken changes around once every 10 seconds.
-                        // If we don't update it on empty events, we do keep consistency, but resuming the stream
-                        // with old tokens may cause connection timeouts.
-                        // We throttle this further by only persisting a keepalive once a minute.
-                        // We add an additional check for waitForCheckpointLsn == null, to make sure we're not
-                        // doing a keepalive in the middle of a transaction.
-                        if (waitForCheckpointLsn == null && performance.now() - lastEmptyResume > 60_000) {
-                            const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(stream.resumeToken);
-                            await batch.keepalive(lsn);
-                            this.touch();
-                            lastEmptyResume = performance.now();
-                            // Log the token update. This helps as a general "replication is still active" message in the logs.
-                            // This token would typically be around 10s behind.
-                            this.logger.info(`Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`);
-                            this.isStartingReplication = false;
-                        }
+                    if (splitEvent.fragment == splitEvent.of) {
+                        // Got all fragments
+                        changeDocument = splitDocument;
+                        splitDocument = null;
+                    }
+                    else {
+                        // Wait for more fragments
                         continue;
                     }
-                    this.touch();
-                    if (startAfter != null && originalChangeDocument.clusterTime?.lte(startAfter)) {
+                }
+                else if (splitDocument != null) {
+                    // We were waiting for fragments, but got a different event
+                    throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+                }
+                if (!filters.multipleDatabases &&
+                    'ns' in changeDocument &&
+                    changeDocument.ns.db != this.defaultDb.databaseName &&
+                    changeDocument.ns.db.endsWith(`_${this.defaultDb.databaseName}`)) {
+                    // When all of the following conditions are met:
+                    // 1. We're replicating from an Atlas Flex instance.
+                    // 2. There were changestream events recorded while the PowerSync service is paused.
+                    // 3. We're only replicating from a single database.
+                    // Then we've observed an ns with for example {db: '67b83e86cd20730f1e766dde_ps'},
+                    // instead of the expected {db: 'ps'}.
+                    // We correct this.
+                    changeDocument.ns.db = this.defaultDb.databaseName;
+                    if (!flexDbNameWorkaroundLogged) {
+                        flexDbNameWorkaroundLogged = true;
+                        this.logger.warn(`Incorrect DB name in change stream: ${changeDocument.ns.db}. Changed to ${this.defaultDb.databaseName}.`);
+                    }
+                }
+                const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
+                if (ns?.coll == CHECKPOINTS_COLLECTION) {
+                    /**
+                     * Dropping the database does not provide an `invalidate` event.
+                     * We typically would receive `drop` events for the collection which we
+                     * would process below.
+                     *
+                     * However we don't commit the LSN after collections are dropped.
+                     * The prevents the `startAfter` or `resumeToken` from advancing past the drop events.
+                     * The stream also closes after the drop events.
+                     * This causes an infinite loop of processing the collection drop events.
+                     *
+                     * This check here invalidates the change stream if our `_checkpoints` collection
+                     * is dropped. This allows for detecting when the DB is dropped.
+                     */
+                    if (changeDocument.operationType == 'drop') {
+                        throw new ChangeStreamInvalidatedError('Internal collections have been dropped', new Error('_checkpoints collection was dropped'));
+                    }
+                    if (!(changeDocument.operationType == 'insert' ||
+                        changeDocument.operationType == 'update' ||
+                        changeDocument.operationType == 'replace')) {
                         continue;
                     }
-                    let changeDocument = originalChangeDocument;
-                    if (originalChangeDocument?.splitEvent != null) {
-                        // Handle split events from $changeStreamSplitLargeEvent.
-                        // This is only relevant for very large update operations.
-                        const splitEvent = originalChangeDocument?.splitEvent;
-                        if (splitDocument == null) {
-                            splitDocument = originalChangeDocument;
-                        }
-                        else {
-                            splitDocument = Object.assign(splitDocument, originalChangeDocument);
-                        }
-                        if (splitEvent.fragment == splitEvent.of) {
-                            // Got all fragments
-                            changeDocument = splitDocument;
-                            splitDocument = null;
-                        }
-                        else {
-                            // Wait for more fragments
+                    // We handle two types of checkpoint events:
+                    // 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
+                    //    immediately, regardless of where they were created.
+                    // 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
+                    //    limiting of commits, so we specifically want to exclude checkpoints from other streams.
+                    //
+                    // It may be useful to also throttle commits due to standalone checkpoints in the future.
+                    // However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
+                    const checkpointId = changeDocument.documentKey._id;
+                    if (checkpointId == STANDALONE_CHECKPOINT_ID) {
+                        // Standalone / write checkpoint received.
+                        // When we are caught up, commit immediately to keep write checkpoint latency low.
+                        // Once there is already a batch checkpoint pending, or the driver has buffered more
+                        // change stream events, collapse standalone checkpoints into the normal batch
+                        // checkpoint flow to avoid commit churn under sustained load.
+                        if (waitForCheckpointLsn != null || this.getBufferedChangeCount(stream) > 0) {
+                            if (waitForCheckpointLsn == null) {
+                                waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                            }
                             continue;
                         }
                     }
-                    else if (splitDocument != null) {
-                        // We were waiting for fragments, but got a different event
-                        throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
+                    else if (!this.checkpointStreamId.equals(checkpointId)) {
+                        continue;
                     }
-                    if (!filters.multipleDatabases &&
-                        'ns' in changeDocument &&
-                        changeDocument.ns.db != this.defaultDb.databaseName &&
-                        changeDocument.ns.db.endsWith(`_${this.defaultDb.databaseName}`)) {
-                        // When all of the following conditions are met:
-                        // 1. We're replicating from an Atlas Flex instance.
-                        // 2. There were changestream events recorded while the PowerSync service is paused.
-                        // 3. We're only replicating from a single database.
-                        // Then we've observed an ns with for example {db: '67b83e86cd20730f1e766dde_ps'},
-                        // instead of the expected {db: 'ps'}.
-                        // We correct this.
-                        changeDocument.ns.db = this.defaultDb.databaseName;
-                        if (!flexDbNameWorkaroundLogged) {
-                            flexDbNameWorkaroundLogged = true;
-                            this.logger.warn(`Incorrect DB name in change stream: ${changeDocument.ns.db}. Changed to ${this.defaultDb.databaseName}.`);
-                        }
+                    const { comparable: lsn } = new MongoLSN({
+                        timestamp: changeDocument.clusterTime,
+                        resume_token: changeDocument._id
+                    });
+                    if (batch.lastCheckpointLsn != null && lsn < batch.lastCheckpointLsn) {
+                        // Checkpoint out of order - should never happen with MongoDB.
+                        // If it does happen, we throw an error to stop the replication - restarting should recover.
+                        // Since we use batch.lastCheckpointLsn for the next resumeAfter, this should not result in an infinite loop.
+                        // Originally a workaround for https://jira.mongodb.org/browse/NODE-7042.
+                        // This has been fixed in the driver in the meantime, but we still keep this as a safety-check.
+                        throw new ReplicationAssertionError(`Change resumeToken ${changeDocument._id._data} (${timestampToDate(changeDocument.clusterTime).toISOString()}) is less than last checkpoint LSN ${batch.lastCheckpointLsn}. Restarting replication.`);
                     }
-                    const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
-                    if (ns?.coll == CHECKPOINTS_COLLECTION) {
-                        /**
-                         * Dropping the database does not provide an `invalidate` event.
-                         * We typically would receive `drop` events for the collection which we
-                         * would process below.
-                         *
-                         * However we don't commit the LSN after collections are dropped.
-                         * The prevents the `startAfter` or `resumeToken` from advancing past the drop events.
-                         * The stream also closes after the drop events.
-                         * This causes an infinite loop of processing the collection drop events.
-                         *
-                         * This check here invalidates the change stream if our `_checkpoints` collection
-                         * is dropped. This allows for detecting when the DB is dropped.
-                         */
-                        if (changeDocument.operationType == 'drop') {
-                            throw new ChangeStreamInvalidatedError('Internal collections have been dropped', new Error('_checkpoints collection was dropped'));
-                        }
-                        if (!(changeDocument.operationType == 'insert' ||
-                            changeDocument.operationType == 'update' ||
-                            changeDocument.operationType == 'replace')) {
-                            continue;
-                        }
-                        // We handle two types of checkpoint events:
-                        // 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
-                        //    immediately, regardless of where they were created.
-                        // 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
-                        //    limiting of commits, so we specifically want to exclude checkpoints from other streams.
-                        //
-                        // It may be useful to also throttle commits due to standalone checkpoints in the future.
-                        // However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
-                        const checkpointId = changeDocument.documentKey._id;
-                        if (!(checkpointId == STANDALONE_CHECKPOINT_ID || this.checkpointStreamId.equals(checkpointId))) {
-                            continue;
-                        }
-                        const { comparable: lsn } = new MongoLSN({
-                            timestamp: changeDocument.clusterTime,
-                            resume_token: changeDocument._id
-                        });
-                        if (batch.lastCheckpointLsn != null && lsn < batch.lastCheckpointLsn) {
-                            // Checkpoint out of order - should never happen with MongoDB.
-                            // If it does happen, we throw an error to stop the replication - restarting should recover.
-                            // Since we use batch.lastCheckpointLsn for the next resumeAfter, this should not result in an infinite loop.
-                            // Originally a workaround for https://jira.mongodb.org/browse/NODE-7042.
-                            // This has been fixed in the driver in the meantime, but we still keep this as a safety-check.
-                            throw new ReplicationAssertionError(`Change resumeToken ${changeDocument._id._data} (${timestampToDate(changeDocument.clusterTime).toISOString()}) is less than last checkpoint LSN ${batch.lastCheckpointLsn}. Restarting replication.`);
-                        }
-                        if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
-                            waitForCheckpointLsn = null;
-                        }
-                        const { checkpointBlocked } = await batch.commit(lsn, {
-                            oldestUncommittedChange: this.oldestUncommittedChange
-                        });
-                        if (!checkpointBlocked) {
-                            this.oldestUncommittedChange = null;
-                            this.isStartingReplication = false;
-                            changesSinceLastCheckpoint = 0;
-                        }
+                    if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
+                        waitForCheckpointLsn = null;
                     }
-                    else if (changeDocument.operationType == 'insert' ||
-                        changeDocument.operationType == 'update' ||
-                        changeDocument.operationType == 'replace' ||
-                        changeDocument.operationType == 'delete') {
-                        if (waitForCheckpointLsn == null) {
-                            waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                    const { checkpointBlocked } = await batch.commit(lsn, {
+                        oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
+                    });
+                    if (!checkpointBlocked) {
+                        this.replicationLag.markCommitted();
+                        changesSinceLastCheckpoint = 0;
+                    }
+                }
+                else if (changeDocument.operationType == 'insert' ||
+                    changeDocument.operationType == 'update' ||
+                    changeDocument.operationType == 'replace' ||
+                    changeDocument.operationType == 'delete') {
+                    if (waitForCheckpointLsn == null) {
+                        waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
+                    }
+                    const rel = getMongoRelation(changeDocument.ns);
+                    const table = await this.getRelation(batch, rel, {
+                        // In most cases, we should not need to snapshot this. But if this is the first time we see the collection
+                        // for whatever reason, then we do need to snapshot it.
+                        // This may result in some duplicate operations when a collection is created for the first time after
+                        // sync rules was deployed.
+                        snapshot: true
+                    });
+                    if (table.syncAny) {
+                        this.replicationLag.trackUncommittedChange(changeDocument.clusterTime == null ? null : timestampToDate(changeDocument.clusterTime));
+                        const transactionKeyValue = transactionKey(changeDocument);
+                        if (transactionKeyValue == null || lastTxnKey != transactionKeyValue) {
+                            // Very crude metric for counting transactions replicated.
+                            // We ignore operations other than basic CRUD, and ignore changes to _powersync_checkpoints.
+                            // Individual writes may not have a txnNumber, in which case we count them as separate transactions.
+                            lastTxnKey = transactionKeyValue;
+                            transactionsReplicatedMetric.add(1);
                         }
-                        const rel = getMongoRelation(changeDocument.ns);
-                        const table = await this.getRelation(batch, rel, {
-                            // In most cases, we should not need to snapshot this. But if this is the first time we see the collection
-                            // for whatever reason, then we do need to snapshot it.
-                            // This may result in some duplicate operations when a collection is created for the first time after
-                            // sync rules was deployed.
-                            snapshot: true
-                        });
-                        if (table.syncAny) {
-                            if (this.oldestUncommittedChange == null && changeDocument.clusterTime != null) {
-                                this.oldestUncommittedChange = timestampToDate(changeDocument.clusterTime);
-                            }
-                            const flushResult = await this.writeChange(batch, table, changeDocument);
-                            changesSinceLastCheckpoint += 1;
-                            if (flushResult != null && changesSinceLastCheckpoint >= 20_000) {
-                                // When we are catching up replication after an initial snapshot, there may be a very long delay
-                                // before we do a commit(). In that case, we need to periodically persist the resume LSN, so
-                                // we don't restart from scratch if we restart replication.
-                                // The same could apply if we need to catch up on replication after some downtime.
-                                const { comparable: lsn } = new MongoLSN({
-                                    timestamp: changeDocument.clusterTime,
-                                    resume_token: changeDocument._id
-                                });
-                                this.logger.info(`Updating resume LSN to ${lsn} after ${changesSinceLastCheckpoint} changes`);
-                                await batch.setResumeLsn(lsn);
-                                changesSinceLastCheckpoint = 0;
-                            }
+                        const flushResult = await this.writeChange(batch, table, changeDocument);
+                        changesSinceLastCheckpoint += 1;
+                        if (flushResult != null && changesSinceLastCheckpoint >= 20_000) {
+                            // When we are catching up replication after an initial snapshot, there may be a very long delay
+                            // before we do a commit(). In that case, we need to periodically persist the resume LSN, so
+                            // we don't restart from scratch if we restart replication.
+                            // The same could apply if we need to catch up on replication after some downtime.
+                            const { comparable: lsn } = new MongoLSN({
+                                timestamp: changeDocument.clusterTime,
+                                resume_token: changeDocument._id
+                            });
+                            this.logger.info(`Updating resume LSN to ${lsn} after ${changesSinceLastCheckpoint} changes`);
+                            await batch.setResumeLsn(lsn);
+                            changesSinceLastCheckpoint = 0;
                         }
                     }
-                    else if (changeDocument.operationType == 'drop') {
-                        const rel = getMongoRelation(changeDocument.ns);
-                        const table = await this.getRelation(batch, rel, {
-                            // We're "dropping" this collection, so never snapshot it.
-                            snapshot: false
-                        });
-                        if (table.syncAny) {
-                            await batch.drop([table]);
-                            this.relationCache.delete(table);
-                        }
+                }
+                else if (changeDocument.operationType == 'drop') {
+                    const rel = getMongoRelation(changeDocument.ns);
+                    const table = await this.getRelation(batch, rel, {
+                        // We're "dropping" this collection, so never snapshot it.
+                        snapshot: false
+                    });
+                    if (table.syncAny) {
+                        await batch.drop([table]);
+                        this.relationCache.delete(table);
                     }
-                    else if (changeDocument.operationType == 'rename') {
-                        const relFrom = getMongoRelation(changeDocument.ns);
-                        const relTo = getMongoRelation(changeDocument.to);
-                        const tableFrom = await this.getRelation(batch, relFrom, {
-                            // We're "dropping" this collection, so never snapshot it.
-                            snapshot: false
-                        });
-                        if (tableFrom.syncAny) {
-                            await batch.drop([tableFrom]);
-                            this.relationCache.delete(relFrom);
-                        }
-                        // Here we do need to snapshot the new table
-                        const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
-                        await this.handleRelation(batch, relTo, {
-                            // This is a new (renamed) collection, so always snapshot it.
-                            snapshot: true,
-                            collectionInfo: collection
-                        });
+                }
+                else if (changeDocument.operationType == 'rename') {
+                    const relFrom = getMongoRelation(changeDocument.ns);
+                    const relTo = getMongoRelation(changeDocument.to);
+                    const tableFrom = await this.getRelation(batch, relFrom, {
+                        // We're "dropping" this collection, so never snapshot it.
+                        snapshot: false
+                    });
+                    if (tableFrom.syncAny) {
+                        await batch.drop([tableFrom]);
+                        this.relationCache.delete(relFrom);
                     }
+                    // Here we do need to snapshot the new table
+                    const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
+                    await this.handleRelation(batch, relTo, {
+                        // This is a new (renamed) collection, so always snapshot it.
+                        snapshot: true,
+                        collectionInfo: collection
+                    });
                 }
             }
-            catch (e_4) {
-                env_4.error = e_4;
-                env_4.hasError = true;
-            }
-            finally {
-                const result_4 = __disposeResources(env_4);
-                if (result_4)
-                    await result_4;
-            }
         });
     }
-    async getReplicationLagMillis() {
-        if (this.oldestUncommittedChange == null) {
-            if (this.isStartingReplication) {
-                // We don't have anything to compute replication lag with yet.
-                return undefined;
-            }
-            else {
-                // We don't have any uncommitted changes, so replication is up-to-date.
-                return 0;
-            }
-        }
-        return Date.now() - this.oldestUncommittedChange.getTime();
+    getReplicationLagMillis() {
+        return this.replicationLag.getLagMillis();
     }
     lastTouchedAt = performance.now();
     touch() {
@@ -1023,4 +949,13 @@ function mapChangeStreamError(e) {
         throw new DatabaseConnectionError(ErrorCode.PSYNC_S1346, `Error reading MongoDB ChangeStream`, e);
     }
 }
+/**
+ * Transaction key for a change stream event, used to detect transaction boundaries. Returns null if the event is not part of a transaction.
+ */
+function transactionKey(doc) {
+    if (doc.txnNumber == null || doc.lsid == null) {
+        return null;
+    }
+    return `${doc.lsid.id.toString('hex')}:${doc.txnNumber}`;
+}
 //# sourceMappingURL=ChangeStream.js.map