@powersync/service-module-mongodb 0.15.4 → 0.17.0

package/test/src/change_stream.test.ts

@@ -3,7 +3,7 @@ import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test, vi } from 'vitest';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { createWriteCheckpoint } from '@powersync/service-core';
+import { createWriteCheckpoint, storage } from '@powersync/service-core';
 import { test_utils } from '@powersync/service-core-tests';
 
 import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
@@ -23,6 +23,8 @@ describe('change stream', () => {
 });
 
 function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const supportsConcurrentSnapshots = storageVersion >= 3;
+
   const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
     return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
   };
@@ -94,6 +96,77 @@ bucket_definitions:
     ]);
   });
 
+  test('does not resurrect rows deleted while snapshotting', async () => {
+    // Special case for testing:
+    // 1. Row A exists.
+    // 2. Start a snapshot and streaming.
+    // 3. Row A is deleted.
+    // 4. Streaming sees the delete, writes the delete.
+    // 5. Snapshot still sees row A and writes it.
+    // The streaming delete should win, using skipExistingRows. For that to work, we rely on soft deletes.
+    // To trigger this case, we need the delete to trigger and process the delete in between the snapshot read and the snapshot write, which we can do using beforeBatchFlush.
+
+    let collection!: mongo.Collection;
+    let testId!: mongo.ObjectId;
+    let interceptedSnapshotFlush = false;
+    let sourceDeleteWritten = false;
+    let sourceDeleteFlushBatch: storage.BucketStorageBatch | undefined;
+    const streamDeleteFlushed = Promise.withResolvers<void>();
+
+    await using context = await openContext({
+      streamOptions: {
+        snapshotChunkLength: 1,
+        storageHooks: {
+          beforeBatchFlush: async (batch) => {
+            if (!batch.skipExistingRows && sourceDeleteWritten && sourceDeleteFlushBatch == null) {
+              sourceDeleteFlushBatch = batch;
+            }
+
+            // skipExistingRows means we're busy with snapshotting.
+            if (!batch.skipExistingRows || interceptedSnapshotFlush) {
+              return;
+            }
+
+            interceptedSnapshotFlush = true;
+            await collection.deleteOne({ _id: testId });
+            sourceDeleteWritten = true;
+            if (!supportsConcurrentSnapshots) {
+              return;
+            }
+            await Promise.race([
+              streamDeleteFlushed.promise,
+              setTimeout(10_000).then(() => {
+                throw new Error('Timed out waiting for streamed delete to flush');
+              })
+            ]);
+          },
+          afterBatchFlush: async (batch) => {
+            if (batch == sourceDeleteFlushBatch) {
+              streamDeleteFlushed.resolve();
+            }
+          }
+        }
+      }
+    });
+    const { db } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await db.createCollection('test_data', {
+      changeStreamPreAndPostImages: { enabled: false }
+    });
+    testId = new mongo.ObjectId();
+    collection = db.collection('test_data');
+    await collection.insertOne({ _id: testId, description: 'stale snapshot row' });
+
+    await context.replicateSnapshot();
+
+    expect(interceptedSnapshotFlush).toBe(true);
+    await context.getCheckpoint();
+
+    const data = await context.getBucketData('global[]');
+    expect(test_utils.reduceBucket(data).slice(1)).toEqual([]);
+  });
+
   test('updateLookup - no fullDocument available', async () => {
     await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.OFF }
@@ -257,6 +330,53 @@ bucket_definitions:
     expect(data).toMatchObject([test_utils.putOp('test_DATA', { id: test_id, description: 'test1' })]);
   });
 
+  test('replicating from multiple databases in the same cluster', async () => {
+    await using context = await openContext();
+    const { client, db } = context;
+    const otherDb = client.db(`${db.databaseName}_other_${storageVersion}`);
+    await otherDb.dropDatabase();
+    await using _ = {
+      [Symbol.asyncDispose]: async () => {
+        await otherDb.dropDatabase();
+      }
+    };
+
+    await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT _id as id, description FROM "${db.databaseName}"."test_data_default"
+            - SELECT _id as id, description FROM "${otherDb.databaseName}"."test_data_other"
+      `);
+
+    await db.createCollection('test_data_default');
+    await otherDb.createCollection('test_data_other');
+    await context.replicateSnapshot();
+    context.startStreaming();
+
+    const defaultResult = await db.collection('test_data_default').insertOne({ description: 'default db' });
+    const otherResult = await otherDb.collection('test_data_other').insertOne({ description: 'other db' });
+
+    const data = await context.getBucketData('global[]');
+
+    expect(data).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining(
+          test_utils.putOp('test_data_default', {
+            id: defaultResult.insertedId.toHexString(),
+            description: 'default db'
+          })
+        ),
+        expect.objectContaining(
+          test_utils.putOp('test_data_other', {
+            id: otherResult.insertedId.toHexString(),
+            description: 'other db'
+          })
+        )
+      ])
+    );
+  });
+
   test('replicating large values', async () => {
     await using context = await openContext();
     const { db } = context;
@@ -351,6 +471,119 @@ bucket_definitions:
     ]);
   });
 
+  test.runIf(supportsConcurrentSnapshots)(
+    'collection recreated while queued snapshot is waiting does not stall checkpoints',
+    async () => {
+      // This is a regression test for a specific timing issue in concurrent snapshot logic.
+      // Regression flow:
+      // 1. Create test_b and wait until streaming has queued a concurrent snapshot for its source table.
+      // 2. Pause that queued snapshot before it reads storage state.
+      // 3. Drop test_b and wait until streaming has deleted the queued source table row from storage.
+      // 4. Recreate the test_b collection, but do not insert replacement data yet. This leaves the collection
+      //    visible to namespace-based resolution without giving normal streaming an insert event that could
+      //    discover and queue the replacement source table.
+      // 5. Release the stale queued snapshot and wait for a checkpoint. The old implementation re-resolved by
+      //    namespace here, created a new source table row, then skipped it because its id did not match the
+      //    queued id. That left snapshot_done=false with no queue item to finish it, stalling checkpoints.
+      // 6. After the checkpoint proves the stale snapshot did not orphan a source table, insert replacement data
+      //    and verify normal streaming still replicates it.
+      const testBSnapshotStarted = Promise.withResolvers<void>();
+      const releaseTestBSnapshot = Promise.withResolvers<void>();
+      let pausedTestBSnapshot = false;
+      let queuedTestBTable: storage.SourceTable | null = null;
+      let waitForStreamingFlush: PromiseWithResolvers<void> | null = null;
+      const waitFor = async (promise: Promise<void>, description: string) => {
+        await Promise.race([
+          promise,
+          setTimeout(10_000).then(() => {
+            throw new Error(`Timed out waiting for ${description}`);
+          })
+        ]);
+      };
+
+      await using context = await openContext({
+        streamOptions: {
+          snapshotChunkLength: 1,
+          snapshotHooks: {
+            beforeSnapshotStarted: async (table) => {
+              if (table.name != 'test_b' || pausedTestBSnapshot) {
+                return;
+              }
+
+              queuedTestBTable = table;
+              pausedTestBSnapshot = true;
+              testBSnapshotStarted.resolve();
+              await releaseTestBSnapshot.promise;
+            }
+          },
+          storageHooks: {
+            afterBatchFlush: async (batch) => {
+              if (batch.skipExistingRows) {
+                return;
+              }
+
+              waitForStreamingFlush?.resolve();
+            }
+          }
+        }
+      });
+      const { db } = context;
+      const waitForSourceTableDropped = async (table: storage.SourceTable) => {
+        const deadline = Date.now() + 10_000;
+        while (Date.now() < deadline) {
+          await using writer = await context.storage!.createWriter(test_utils.BATCH_OPTIONS);
+          if ((await writer.getSourceTableStatus(table)) == null) {
+            return;
+          }
+          await setTimeout(50);
+        }
+        throw new Error('Timed out waiting for test_b source table to be dropped');
+      };
+      const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_%"
+`;
+      await context.updateSyncRules(syncRuleContent);
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      try {
+        await db.createCollection('test_a');
+        const testA = await db.collection('test_a').insertOne({ description: 'test a' });
+
+        await db.createCollection('test_b');
+        waitForStreamingFlush = Promise.withResolvers<void>();
+        await db.collection('test_b').insertOne({ description: 'old test b' });
+        await waitFor(waitForStreamingFlush.promise, 'old test_b streaming flush');
+        await waitFor(testBSnapshotStarted.promise, 'test_b snapshot to start');
+
+        if (queuedTestBTable == null) {
+          throw new Error('test_b snapshot started without a queued source table');
+        }
+
+        await db.collection('test_b').drop();
+        await waitForSourceTableDropped(queuedTestBTable);
+
+        await db.createCollection('test_b');
+        releaseTestBSnapshot.resolve();
+        await context.getCheckpoint({ timeout: 10_000 });
+
+        const result = await db.collection('test_b').insertOne({ description: 'new test b' });
+
+        const data = await context.getBucketData('global[]', undefined, { timeout: 10_000 });
+        const reduced = test_utils.reduceBucket(data).slice(1);
+        expect(reduced).toMatchObject([
+          test_utils.putOp('test_a', { id: testA.insertedId.toHexString(), description: 'test a' }),
+          test_utils.putOp('test_b', { id: result.insertedId.toHexString(), description: 'new test b' })
+        ]);
+      } finally {
+        releaseTestBSnapshot.resolve();
+      }
+    }
+  );
+
   test('initial sync', async () => {
     await using context = await openContext();
     const { db } = context;
@@ -373,12 +606,6 @@ bucket_definitions:
   test('coalesces standalone checkpoints when backlog is buffered', async () => {
     await using context = await openContext();
     await context.updateSyncRules(BASIC_SYNC_RULES);
-    await context.replicateSnapshot();
-    await context.markSnapshotConsistent();
-    await using api = new MongoRouteAPIAdapter({
-      type: 'mongodb',
-      ...TEST_CONNECTION_OPTIONS
-    });
 
     let commitCount = 0;
     // This relies on internals to count how often checkpoints are committed
@@ -392,6 +619,13 @@ bucket_definitions:
       }
     });
 
+    await context.replicateSnapshot();
+    await context.markSnapshotConsistent();
+    await using api = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS
+    });
+
     context.startStreaming();
 
     // Wait until the stream is active and caught up, then start counting from zero.
@@ -449,7 +683,7 @@ bucket_definitions:
     // The field appears twice in the ChangeStream event, so the total size
     // is > 16MB.
 
-    // We don't actually have this description field in the sync rules,
+    // We don't actually have this description field in the sync config,
     // That causes other issues, not relevant for this specific test.
     const largeDescription = crypto.randomBytes(12000000 / 2).toString('hex');
 
@@ -478,7 +712,7 @@ bucket_definitions:
     });
   });
 
-  test('collection not in sync rules', async () => {
+  test('collection not in sync config', async () => {
     await using context = await openContext();
     const { db } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
@@ -520,12 +754,19 @@ bucket_definitions:
     context.startStreaming();
 
     const data = await context.getBucketData('global[]');
-    expect(data).toMatchObject([
-      // An extra op here, since this triggers a snapshot in addition to getting the event.
-      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
-      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
-      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
-    ]);
+    if (data.length == 3) {
+      expect(data).toMatchObject([
+        // An extra op here, since this triggers a snapshot in addition to getting the event.
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+      ]);
+    } else {
+      expect(data).toMatchObject([
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+      ]);
+    }
   });
 
   test('postImages - new collection with postImages disabled', async () => {
@@ -575,7 +816,7 @@ bucket_definitions:
     await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
 
     await context.replicateSnapshot();
-    await context.markSnapshotConsistent();
+    await context.getCheckpoint();
 
     // Simulate an error
     await context.storage!.reportError(new Error('simulated error'));
@@ -583,10 +824,9 @@ bucket_definitions:
     expect(syncRules).toBeTruthy();
     expect(syncRules?.last_fatal_error).toEqual('simulated error');
 
-    // startStreaming() should automatically clear the error.
-    context.startStreaming();
+    // The next checkpoint should clear the error.
+    await context.getCheckpoint();
 
-    // getBucketData() creates a checkpoint that clears the error, so we don't do that
     // Just wait, and check that the error is cleared automatically.
     await vi.waitUntil(
       async () => {

package/test/src/change_stream_utils.ts

@@ -1,4 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAbortedError } from '@powersync/lib-services-framework';
 import {
   BucketStorageFactory,
   createCoreReplicationMetrics,
@@ -8,10 +9,11 @@ import {
   OplogEntry,
   ProtocolOpId,
   ReplicationCheckpoint,
+  settledPromise,
   storage,
-  STORAGE_VERSION_CONFIG,
   SyncRulesBucketStorage,
   TestStorageOptions,
+  unsettledPromise,
   updateSyncRulesFromYaml,
   utils
 } from '@powersync/service-core';
@@ -27,8 +29,7 @@ import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
   private abortController = new AbortController();
-  private streamPromise?: Promise<PromiseSettledResult<void>>;
-  private syncRulesId?: number;
+  private settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
   private syncRulesContent?: storage.PersistedSyncRulesContent;
   public storage?: SyncRulesBucketStorage;
 
@@ -55,17 +56,15 @@ export class ChangeStreamTestContext {
     }
 
     const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION;
-    const versionedBuckets = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false;
 
-    return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion, versionedBuckets);
+    return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion);
   }
 
   constructor(
     public factory: BucketStorageFactory,
     public connectionManager: MongoManager,
     private streamOptions: Partial<ChangeStreamOptions> = {},
-    private storageVersion: number = LEGACY_STORAGE_VERSION,
-    private versionedBuckets: boolean = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false
+    private storageVersion: number = LEGACY_STORAGE_VERSION
   ) {
     createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
     initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
@@ -74,13 +73,13 @@ export class ChangeStreamTestContext {
   /**
    * Abort snapshot and/or replication, without actively closing connections.
    */
-  abort() {
-    this.abortController.abort();
+  abort(cause?: Error) {
+    this.abortController.abort(cause);
   }
 
   async dispose() {
-    this.abort();
-    await this.streamPromise?.catch((e) => e);
+    this.abort(new Error('Disposing test context'));
+    await this.settledReplicationPromise;
     await this.factory[Symbol.asyncDispose]();
     await this.connectionManager.end();
   }
@@ -105,7 +104,6 @@ export class ChangeStreamTestContext {
     const syncRules = await this.factory.updateSyncRules(
       updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
     );
-    this.syncRulesId = syncRules.id;
     this.syncRulesContent = syncRules;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
@@ -114,10 +112,9 @@ export class ChangeStreamTestContext {
   async loadNextSyncRules() {
     const syncRules = await this.factory.getNextSyncRulesContent();
     if (syncRules == null) {
-      throw new Error(`Next sync rules not available`);
+      throw new Error(`Next sync config not available`);
     }
 
-    this.syncRulesId = syncRules.id;
     this.syncRulesContent = syncRules;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
@@ -125,7 +122,7 @@ export class ChangeStreamTestContext {
 
   private getSyncRulesContent(): storage.PersistedSyncRulesContent {
     if (this.syncRulesContent == null) {
-      throw new Error('Sync rules not configured - call updateSyncRules() first');
+      throw new Error('Sync config not configured - call updateSyncRules() first');
     }
     return this.syncRulesContent;
   }
@@ -145,14 +142,25 @@ export class ChangeStreamTestContext {
       // Specifically reduce this from the default for tests on MongoDB <= 6.0, otherwise it can take
       // a long time to abort the stream.
       maxAwaitTimeMS: this.streamOptions?.maxAwaitTimeMS ?? 200,
-      snapshotChunkLength: this.streamOptions?.snapshotChunkLength
+      snapshotChunkLength: this.streamOptions?.snapshotChunkLength,
+      storageHooks: this.streamOptions?.storageHooks,
+      snapshotHooks: this.streamOptions?.snapshotHooks,
+      logger: this.streamOptions?.logger
     };
     this._walStream = new ChangeStream(options);
     return this._walStream!;
   }
 
   async replicateSnapshot() {
-    await this.streamer.initReplication();
+    this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
+    try {
+      await Promise.race([unsettledPromise(this.settledReplicationPromise), this.streamer.waitForInitialSnapshot()]);
+    } catch (e) {
+      if (e instanceof ReplicationAbortedError && e.cause != null) {
+        throw e.cause;
+      }
+      throw e;
+    }
   }
 
   /**
@@ -170,21 +178,14 @@ export class ChangeStreamTestContext {
   }
 
   startStreaming() {
-    this.streamPromise = this.streamer
-      .streamChanges()
-      .then(() => ({ status: 'fulfilled', value: undefined }) satisfies PromiseFulfilledResult<void>)
-      .catch((reason) => ({ status: 'rejected', reason }) satisfies PromiseRejectedResult);
-    return this.streamPromise;
+    this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
+    return this.settledReplicationPromise;
   }
 
   async getCheckpoint(options?: { timeout?: number }) {
     let checkpoint = await Promise.race([
       getClientCheckpoint(this.client, this.db, this.factory, { timeout: options?.timeout ?? 15_000 }),
-      this.streamPromise?.then((e) => {
-        if (e.status == 'rejected') {
-          throw e.reason;
-        }
-      })
+      this.settledReplicationPromise == null ? undefined : unsettledPromise(this.settledReplicationPromise)
     ]);
     if (checkpoint == null) {
       // This indicates an issue with the test setup - streamingPromise completed instead

package/test/src/checkpoint_retry.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, test } from 'vitest';
+
+import { mongo } from '@powersync/lib-service-mongodb';
+
+import { MongoLSN } from '@module/common/MongoLSN.js';
+import { createCheckpoint } from '@module/replication/MongoRelation.js';
+import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
+
+describe('checkpoint retryable writes', () => {
+  test('returns the persisted checkpoint clusterTime after a retryable write', { repeats: 1 }, async (ctx) => {
+    // This test relies on very specific timing:
+    //
+    // 1. _powersync_checkpoints findAndModify command starts and is sent to the server
+    // 2. command is written to the server
+    // 3. _powersync_checkpoints findAndModify command fails with a retryable error (e.g. connection timeout)
+    // 4. command is retried
+    // 5. another write lands on the server, increasing the server's operationTime
+    // 6. command succeeds with a new operationTime, but without actually writing again (server de-duplicates based on transaction id)
+    //
+    // It is quite difficult to simulate this, even with failCommand. We currently rely on triggering a socket timeout for the first command,
+    // with another write happening right after that.
+
+    const TIMEOUT = 100;
+    const INSERT_COUNT = 5;
+
+    const { db, client } = await connectMongoData({
+      monitorCommands: true,
+      socketTimeoutMS: TIMEOUT,
+      maxPoolSize: 1
+    });
+
+    await using _client = { [Symbol.asyncDispose]: async () => await client.close() };
+    await client.connect();
+    await using failCommand = await requireFailCommand(client, ctx);
+
+    await clearTestDb(db);
+
+    const checkpointId = new mongo.ObjectId();
+    const advanceClusterTime = db.collection('_advance_cluster_time');
+
+    const changeStream = db.watch([], { maxAwaitTimeMS: TIMEOUT * 0.8 });
+    await using _changeStream = { [Symbol.asyncDispose]: async () => await changeStream.close() };
+
+    const checkpointEventPromise = waitForCheckpointEvent(changeStream, checkpointId, INSERT_COUNT).catch((e) =>
+      console.error(e)
+    );
+
+    const commandsStarted: mongo.CommandStartedEvent[] = [];
+
+    client.on('commandStarted', (event) => {
+      commandsStarted.push(event);
+    });
+
+    const DEBUG_COMMANDS = false;
+    if (DEBUG_COMMANDS) {
+      // For debugging: Log all commands
+      client.on('commandStarted', (event) => {
+        console.log('commandStarted:', event);
+      });
+
+      client.on('commandSucceeded', (event) => {
+        console.log('commandSucceeded:', event);
+      });
+
+      client.on('commandFailed', (event) => {
+        console.log('commandFailed:', event);
+      });
+    }
+
+    await failCommand.configure({
+      mode: { times: 1 },
+      data: {
+        failCommands: ['commitTransaction', 'findAndModify'],
+        blockConnection: true,
+        blockTimeMS: TIMEOUT
+      }
+    });
+    const returnedLsnPromise = createCheckpoint(client, db, checkpointId);
+
+    for (let i = 0; i < INSERT_COUNT; i++) {
+      await advanceClusterTime.insertOne({ at: new Date() });
+    }
+
+    const returnedLsn = await returnedLsnPromise;
+    const checkpointEvent = await checkpointEventPromise;
+    if (checkpointEvent == null) {
+      throw new Error('change stream failed');
+    }
+
+    const eventLsn = new MongoLSN({
+      timestamp: checkpointEvent.clusterTime!,
+      resume_token: checkpointEvent._id
+    }).comparable;
+
+    if (DEBUG_COMMANDS) {
+      const commands = commandsStarted
+        .filter((c) => ['findAndModify', 'insert'].includes(c.commandName))
+        .map((c) => c.commandName);
+      // In the failure case, we had findAndModify, insert, findAndModify, potentially with more inserts in between.
+      // With updated behavior, we only have a single findAndModify.
+      console.log('Command order:', commands);
+    }
+
+    expect(eventLsn >= returnedLsn).toBe(true);
+    expect(checkpointEvent.clusterTime).toEqual(MongoLSN.fromSerialized(returnedLsn).timestamp);
+  });
+});
+
+async function waitForCheckpointEvent(
+  changeStream: mongo.ChangeStream,
+  checkpointId: mongo.ObjectId,
+  insertCount: number
+): Promise<mongo.ChangeStreamDocument | null> {
+  let lastEvent: mongo.ChangeStreamDocument | null = null;
+  let seenInserts = 0;
+  for await (const event of changeStream) {
+    if (
+      (event.operationType == 'insert' || event.operationType == 'update' || event.operationType == 'replace') &&
+      checkpointId.equals(event.documentKey._id)
+    ) {
+      lastEvent = event;
+    } else if (event.operationType == 'insert' && event.ns.coll === '_advance_cluster_time') {
+      seenInserts++;
+      if (seenInserts >= insertCount) {
+        // We finish when we hit the expected number of inserts, since the number of checkpoint events are unknown.
+        break;
+      }
+    }
+  }
+  return lastEvent;
+}