@powersync/service-module-mongodb 0.16.0 → 0.17.0

package/test/src/change_stream.test.ts

@@ -3,7 +3,7 @@ import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test, vi } from 'vitest';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { createWriteCheckpoint } from '@powersync/service-core';
+import { createWriteCheckpoint, storage } from '@powersync/service-core';
 import { test_utils } from '@powersync/service-core-tests';
 
 import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
@@ -23,6 +23,8 @@ describe('change stream', () => {
 });
 
 function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const supportsConcurrentSnapshots = storageVersion >= 3;
+
   const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
     return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
   };
@@ -94,6 +96,77 @@ bucket_definitions:
     ]);
   });
 
+  test('does not resurrect rows deleted while snapshotting', async () => {
+    // Special case for testing:
+    // 1. Row A exists.
+    // 2. Start a snapshot and streaming.
+    // 3. Row A is deleted.
+    // 4. Streaming sees the delete, writes the delete.
+    // 5. Snapshot still sees row A and writes it.
+    // The streaming delete should win, using skipExistingRows. For that to work, we rely on soft deletes.
+    // To trigger this case, we need the delete to trigger and process the delete in between the snapshot read and the snapshot write, which we can do using beforeBatchFlush.
+
+    let collection!: mongo.Collection;
+    let testId!: mongo.ObjectId;
+    let interceptedSnapshotFlush = false;
+    let sourceDeleteWritten = false;
+    let sourceDeleteFlushBatch: storage.BucketStorageBatch | undefined;
+    const streamDeleteFlushed = Promise.withResolvers<void>();
+
+    await using context = await openContext({
+      streamOptions: {
+        snapshotChunkLength: 1,
+        storageHooks: {
+          beforeBatchFlush: async (batch) => {
+            if (!batch.skipExistingRows && sourceDeleteWritten && sourceDeleteFlushBatch == null) {
+              sourceDeleteFlushBatch = batch;
+            }
+
+            // skipExistingRows means we're busy with snapshotting.
+            if (!batch.skipExistingRows || interceptedSnapshotFlush) {
+              return;
+            }
+
+            interceptedSnapshotFlush = true;
+            await collection.deleteOne({ _id: testId });
+            sourceDeleteWritten = true;
+            if (!supportsConcurrentSnapshots) {
+              return;
+            }
+            await Promise.race([
+              streamDeleteFlushed.promise,
+              setTimeout(10_000).then(() => {
+                throw new Error('Timed out waiting for streamed delete to flush');
+              })
+            ]);
+          },
+          afterBatchFlush: async (batch) => {
+            if (batch == sourceDeleteFlushBatch) {
+              streamDeleteFlushed.resolve();
+            }
+          }
+        }
+      }
+    });
+    const { db } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await db.createCollection('test_data', {
+      changeStreamPreAndPostImages: { enabled: false }
+    });
+    testId = new mongo.ObjectId();
+    collection = db.collection('test_data');
+    await collection.insertOne({ _id: testId, description: 'stale snapshot row' });
+
+    await context.replicateSnapshot();
+
+    expect(interceptedSnapshotFlush).toBe(true);
+    await context.getCheckpoint();
+
+    const data = await context.getBucketData('global[]');
+    expect(test_utils.reduceBucket(data).slice(1)).toEqual([]);
+  });
+
   test('updateLookup - no fullDocument available', async () => {
     await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.OFF }
@@ -398,6 +471,119 @@ bucket_definitions:
     ]);
   });
 
+  test.runIf(supportsConcurrentSnapshots)(
+    'collection recreated while queued snapshot is waiting does not stall checkpoints',
+    async () => {
+      // This is a regression test for a specific timing issue in concurrent snapshot logic.
+      // Regression flow:
+      // 1. Create test_b and wait until streaming has queued a concurrent snapshot for its source table.
+      // 2. Pause that queued snapshot before it reads storage state.
+      // 3. Drop test_b and wait until streaming has deleted the queued source table row from storage.
+      // 4. Recreate the test_b collection, but do not insert replacement data yet. This leaves the collection
+      //    visible to namespace-based resolution without giving normal streaming an insert event that could
+      //    discover and queue the replacement source table.
+      // 5. Release the stale queued snapshot and wait for a checkpoint. The old implementation re-resolved by
+      //    namespace here, created a new source table row, then skipped it because its id did not match the
+      //    queued id. That left snapshot_done=false with no queue item to finish it, stalling checkpoints.
+      // 6. After the checkpoint proves the stale snapshot did not orphan a source table, insert replacement data
+      //    and verify normal streaming still replicates it.
+      const testBSnapshotStarted = Promise.withResolvers<void>();
+      const releaseTestBSnapshot = Promise.withResolvers<void>();
+      let pausedTestBSnapshot = false;
+      let queuedTestBTable: storage.SourceTable | null = null;
+      let waitForStreamingFlush: PromiseWithResolvers<void> | null = null;
+      const waitFor = async (promise: Promise<void>, description: string) => {
+        await Promise.race([
+          promise,
+          setTimeout(10_000).then(() => {
+            throw new Error(`Timed out waiting for ${description}`);
+          })
+        ]);
+      };
+
+      await using context = await openContext({
+        streamOptions: {
+          snapshotChunkLength: 1,
+          snapshotHooks: {
+            beforeSnapshotStarted: async (table) => {
+              if (table.name != 'test_b' || pausedTestBSnapshot) {
+                return;
+              }
+
+              queuedTestBTable = table;
+              pausedTestBSnapshot = true;
+              testBSnapshotStarted.resolve();
+              await releaseTestBSnapshot.promise;
+            }
+          },
+          storageHooks: {
+            afterBatchFlush: async (batch) => {
+              if (batch.skipExistingRows) {
+                return;
+              }
+
+              waitForStreamingFlush?.resolve();
+            }
+          }
+        }
+      });
+      const { db } = context;
+      const waitForSourceTableDropped = async (table: storage.SourceTable) => {
+        const deadline = Date.now() + 10_000;
+        while (Date.now() < deadline) {
+          await using writer = await context.storage!.createWriter(test_utils.BATCH_OPTIONS);
+          if ((await writer.getSourceTableStatus(table)) == null) {
+            return;
+          }
+          await setTimeout(50);
+        }
+        throw new Error('Timed out waiting for test_b source table to be dropped');
+      };
+      const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_%"
+`;
+      await context.updateSyncRules(syncRuleContent);
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      try {
+        await db.createCollection('test_a');
+        const testA = await db.collection('test_a').insertOne({ description: 'test a' });
+
+        await db.createCollection('test_b');
+        waitForStreamingFlush = Promise.withResolvers<void>();
+        await db.collection('test_b').insertOne({ description: 'old test b' });
+        await waitFor(waitForStreamingFlush.promise, 'old test_b streaming flush');
+        await waitFor(testBSnapshotStarted.promise, 'test_b snapshot to start');
+
+        if (queuedTestBTable == null) {
+          throw new Error('test_b snapshot started without a queued source table');
+        }
+
+        await db.collection('test_b').drop();
+        await waitForSourceTableDropped(queuedTestBTable);
+
+        await db.createCollection('test_b');
+        releaseTestBSnapshot.resolve();
+        await context.getCheckpoint({ timeout: 10_000 });
+
+        const result = await db.collection('test_b').insertOne({ description: 'new test b' });
+
+        const data = await context.getBucketData('global[]', undefined, { timeout: 10_000 });
+        const reduced = test_utils.reduceBucket(data).slice(1);
+        expect(reduced).toMatchObject([
+          test_utils.putOp('test_a', { id: testA.insertedId.toHexString(), description: 'test a' }),
+          test_utils.putOp('test_b', { id: result.insertedId.toHexString(), description: 'new test b' })
+        ]);
+      } finally {
+        releaseTestBSnapshot.resolve();
+      }
+    }
+  );
+
   test('initial sync', async () => {
     await using context = await openContext();
     const { db } = context;
@@ -420,12 +606,6 @@ bucket_definitions:
   test('coalesces standalone checkpoints when backlog is buffered', async () => {
     await using context = await openContext();
     await context.updateSyncRules(BASIC_SYNC_RULES);
-    await context.replicateSnapshot();
-    await context.markSnapshotConsistent();
-    await using api = new MongoRouteAPIAdapter({
-      type: 'mongodb',
-      ...TEST_CONNECTION_OPTIONS
-    });
 
     let commitCount = 0;
     // This relies on internals to count how often checkpoints are committed
@@ -439,6 +619,13 @@ bucket_definitions:
       }
     });
 
+    await context.replicateSnapshot();
+    await context.markSnapshotConsistent();
+    await using api = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS
+    });
+
     context.startStreaming();
 
     // Wait until the stream is active and caught up, then start counting from zero.
@@ -567,12 +754,19 @@ bucket_definitions:
     context.startStreaming();
 
     const data = await context.getBucketData('global[]');
-    expect(data).toMatchObject([
-      // An extra op here, since this triggers a snapshot in addition to getting the event.
-      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
-      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
-      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
-    ]);
+    if (data.length == 3) {
+      expect(data).toMatchObject([
+        // An extra op here, since this triggers a snapshot in addition to getting the event.
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+      ]);
+    } else {
+      expect(data).toMatchObject([
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+      ]);
+    }
   });
 
   test('postImages - new collection with postImages disabled', async () => {
@@ -622,7 +816,7 @@ bucket_definitions:
     await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
 
     await context.replicateSnapshot();
-    await context.markSnapshotConsistent();
+    await context.getCheckpoint();
 
     // Simulate an error
     await context.storage!.reportError(new Error('simulated error'));
@@ -630,10 +824,9 @@ bucket_definitions:
     expect(syncRules).toBeTruthy();
     expect(syncRules?.last_fatal_error).toEqual('simulated error');
 
-    // startStreaming() should automatically clear the error.
-    context.startStreaming();
+    // The next checkpoint should clear the error.
+    await context.getCheckpoint();
 
-    // getBucketData() creates a checkpoint that clears the error, so we don't do that
     // Just wait, and check that the error is cleared automatically.
     await vi.waitUntil(
       async () => {

package/test/src/change_stream_utils.ts

@@ -1,4 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAbortedError } from '@powersync/lib-services-framework';
 import {
   BucketStorageFactory,
   createCoreReplicationMetrics,
@@ -8,9 +9,11 @@ import {
   OplogEntry,
   ProtocolOpId,
   ReplicationCheckpoint,
+  settledPromise,
   storage,
   SyncRulesBucketStorage,
   TestStorageOptions,
+  unsettledPromise,
   updateSyncRulesFromYaml,
   utils
 } from '@powersync/service-core';
@@ -26,7 +29,7 @@ import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
   private abortController = new AbortController();
-  private streamPromise?: Promise<PromiseSettledResult<void>>;
+  private settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
   private syncRulesContent?: storage.PersistedSyncRulesContent;
   public storage?: SyncRulesBucketStorage;
 
@@ -70,13 +73,13 @@ export class ChangeStreamTestContext {
   /**
    * Abort snapshot and/or replication, without actively closing connections.
    */
-  abort() {
-    this.abortController.abort();
+  abort(cause?: Error) {
+    this.abortController.abort(cause);
   }
 
   async dispose() {
-    this.abort();
-    await this.streamPromise?.catch((e) => e);
+    this.abort(new Error('Disposing test context'));
+    await this.settledReplicationPromise;
     await this.factory[Symbol.asyncDispose]();
     await this.connectionManager.end();
   }
@@ -139,14 +142,25 @@ export class ChangeStreamTestContext {
       // Specifically reduce this from the default for tests on MongoDB <= 6.0, otherwise it can take
       // a long time to abort the stream.
       maxAwaitTimeMS: this.streamOptions?.maxAwaitTimeMS ?? 200,
-      snapshotChunkLength: this.streamOptions?.snapshotChunkLength
+      snapshotChunkLength: this.streamOptions?.snapshotChunkLength,
+      storageHooks: this.streamOptions?.storageHooks,
+      snapshotHooks: this.streamOptions?.snapshotHooks,
+      logger: this.streamOptions?.logger
     };
     this._walStream = new ChangeStream(options);
     return this._walStream!;
   }
 
   async replicateSnapshot() {
-    await this.streamer.initReplication();
+    this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
+    try {
+      await Promise.race([unsettledPromise(this.settledReplicationPromise), this.streamer.waitForInitialSnapshot()]);
+    } catch (e) {
+      if (e instanceof ReplicationAbortedError && e.cause != null) {
+        throw e.cause;
+      }
+      throw e;
+    }
   }
 
   /**
@@ -164,21 +178,14 @@ export class ChangeStreamTestContext {
   }
 
   startStreaming() {
-    this.streamPromise = this.streamer
-      .streamChanges()
-      .then(() => ({ status: 'fulfilled', value: undefined }) satisfies PromiseFulfilledResult<void>)
-      .catch((reason) => ({ status: 'rejected', reason }) satisfies PromiseRejectedResult);
-    return this.streamPromise;
+    this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
+    return this.settledReplicationPromise;
   }
 
   async getCheckpoint(options?: { timeout?: number }) {
     let checkpoint = await Promise.race([
       getClientCheckpoint(this.client, this.db, this.factory, { timeout: options?.timeout ?? 15_000 }),
-      this.streamPromise?.then((e) => {
-        if (e.status == 'rejected') {
-          throw e.reason;
-        }
-      })
+      this.settledReplicationPromise == null ? undefined : unsettledPromise(this.settledReplicationPromise)
     ]);
     if (checkpoint == null) {
       // This indicates an issue with the test setup - streamingPromise completed instead

package/test/src/checkpoint_retry.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, test } from 'vitest';
+
+import { mongo } from '@powersync/lib-service-mongodb';
+
+import { MongoLSN } from '@module/common/MongoLSN.js';
+import { createCheckpoint } from '@module/replication/MongoRelation.js';
+import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
+
+describe('checkpoint retryable writes', () => {
+  test('returns the persisted checkpoint clusterTime after a retryable write', { repeats: 1 }, async (ctx) => {
+    // This test relies on very specific timing:
+    //
+    // 1. _powersync_checkpoints findAndModify command starts and is sent to the server
+    // 2. command is written to the server
+    // 3. _powersync_checkpoints findAndModify command fails with a retryable error (e.g. connection timeout)
+    // 4. command is retried
+    // 5. another write lands on the server, increasing the server's operationTime
+    // 6. command succeeds with a new operationTime, but without actually writing again (server de-duplicates based on transaction id)
+    //
+    // It is quite difficult to simulate this, even with failCommand. We currently rely on triggering a socket timeout for the first command,
+    // with another write happening right after that.
+
+    const TIMEOUT = 100;
+    const INSERT_COUNT = 5;
+
+    const { db, client } = await connectMongoData({
+      monitorCommands: true,
+      socketTimeoutMS: TIMEOUT,
+      maxPoolSize: 1
+    });
+
+    await using _client = { [Symbol.asyncDispose]: async () => await client.close() };
+    await client.connect();
+    await using failCommand = await requireFailCommand(client, ctx);
+
+    await clearTestDb(db);
+
+    const checkpointId = new mongo.ObjectId();
+    const advanceClusterTime = db.collection('_advance_cluster_time');
+
+    const changeStream = db.watch([], { maxAwaitTimeMS: TIMEOUT * 0.8 });
+    await using _changeStream = { [Symbol.asyncDispose]: async () => await changeStream.close() };
+
+    const checkpointEventPromise = waitForCheckpointEvent(changeStream, checkpointId, INSERT_COUNT).catch((e) =>
+      console.error(e)
+    );
+
+    const commandsStarted: mongo.CommandStartedEvent[] = [];
+
+    client.on('commandStarted', (event) => {
+      commandsStarted.push(event);
+    });
+
+    const DEBUG_COMMANDS = false;
+    if (DEBUG_COMMANDS) {
+      // For debugging: Log all commands
+      client.on('commandStarted', (event) => {
+        console.log('commandStarted:', event);
+      });
+
+      client.on('commandSucceeded', (event) => {
+        console.log('commandSucceeded:', event);
+      });
+
+      client.on('commandFailed', (event) => {
+        console.log('commandFailed:', event);
+      });
+    }
+
+    await failCommand.configure({
+      mode: { times: 1 },
+      data: {
+        failCommands: ['commitTransaction', 'findAndModify'],
+        blockConnection: true,
+        blockTimeMS: TIMEOUT
+      }
+    });
+    const returnedLsnPromise = createCheckpoint(client, db, checkpointId);
+
+    for (let i = 0; i < INSERT_COUNT; i++) {
+      await advanceClusterTime.insertOne({ at: new Date() });
+    }
+
+    const returnedLsn = await returnedLsnPromise;
+    const checkpointEvent = await checkpointEventPromise;
+    if (checkpointEvent == null) {
+      throw new Error('change stream failed');
+    }
+
+    const eventLsn = new MongoLSN({
+      timestamp: checkpointEvent.clusterTime!,
+      resume_token: checkpointEvent._id
+    }).comparable;
+
+    if (DEBUG_COMMANDS) {
+      const commands = commandsStarted
+        .filter((c) => ['findAndModify', 'insert'].includes(c.commandName))
+        .map((c) => c.commandName);
+      // In the failure case, we had findAndModify, insert, findAndModify, potentially with more inserts in between.
+      // With updated behavior, we only have a single findAndModify.
+      console.log('Command order:', commands);
+    }
+
+    expect(eventLsn >= returnedLsn).toBe(true);
+    expect(checkpointEvent.clusterTime).toEqual(MongoLSN.fromSerialized(returnedLsn).timestamp);
+  });
+});
+
+async function waitForCheckpointEvent(
+  changeStream: mongo.ChangeStream,
+  checkpointId: mongo.ObjectId,
+  insertCount: number
+): Promise<mongo.ChangeStreamDocument | null> {
+  let lastEvent: mongo.ChangeStreamDocument | null = null;
+  let seenInserts = 0;
+  for await (const event of changeStream) {
+    if (
+      (event.operationType == 'insert' || event.operationType == 'update' || event.operationType == 'replace') &&
+      checkpointId.equals(event.documentKey._id)
+    ) {
+      lastEvent = event;
+    } else if (event.operationType == 'insert' && event.ns.coll === '_advance_cluster_time') {
+      seenInserts++;
+      if (seenInserts >= insertCount) {
+        // We finish when we hit the expected number of inserts, since the number of checkpoint events are unknown.
+        break;
+      }
+    }
+  }
+  return lastEvent;
+}

package/test/src/resuming_snapshots.test.ts

@@ -126,26 +126,30 @@ async function testResumingReplication(factory: TestStorageFactory, storageVersi
     // We only test the final version.
     expect(JSON.parse(updatedRowOps[1].data as string).description).toEqual('update1');
 
-    expect(insertedRowOps.length).toEqual(2);
     expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1');
-    expect(JSON.parse(insertedRowOps[1].data as string).description).toEqual('insert1');
+    if (insertedRowOps.length != 1) {
+      // Also valid.
+      expect(insertedRowOps.length).toEqual(2);
+      expect(JSON.parse(insertedRowOps[1].data as string).description).toEqual('insert1');
+    }
 
     // 1000 of test_data1 during first replication attempt.
     // N >= 1000 of test_data2 during first replication attempt.
     // 10000 - N - 1 + 1 of test_data2 during second replication attempt.
     // An additional update during streaming replication (2x total for this row).
-    // An additional insert during streaming replication (2x total for this row).
+    // An additional insert during streaming replication (1x or 2x total for this row).
     // If the deleted row was part of the first replication batch, it's removed by streaming replication.
     // This adds 2 ops.
     // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000.
     // However, this is not deterministic.
-    const expectedCount = 11002 + deletedRowOps.length;
+    const expectedCount = 11000 + deletedRowOps.length + insertedRowOps.length;
     expect(data.length).toEqual(expectedCount);
 
     const replicatedCount =
       ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
 
-    // With resumable replication, there should be no need to re-replicate anything.
-    expect(replicatedCount).toEqual(expectedCount);
+    // With resumable replication, there should be no need to re-replicate anything, apart from the newly-inserted row.
+    expect(replicatedCount).toBeGreaterThanOrEqual(expectedCount);
+    expect(replicatedCount).toBeLessThanOrEqual(expectedCount + 1);
   }
 }