@powersync/service-module-mongodb 0.9.0 → 0.10.0

package/test/src/resume.test.ts

@@ -1,96 +1,14 @@
-import { MongoLSN, ZERO_LSN } from '@module/common/MongoLSN.js';
-
+import { ChangeStreamInvalidatedError } from '@module/replication/ChangeStream.js';
 import { MongoManager } from '@module/replication/MongoManager.js';
 import { normalizeConnectionConfig } from '@module/types/types.js';
-import { isMongoServerError, mongo } from '@powersync/lib-service-mongodb';
 import { BucketStorageFactory, TestStorageOptions } from '@powersync/service-core';
-import { describe, expect, test, vi } from 'vitest';
+import { describe, expect, test } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
 import { env } from './env.js';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
-import { ChangeStreamInvalidatedError } from '@module/replication/ChangeStream.js';
-
-describe('mongo lsn', () => {
-  test('LSN with resume tokens should be comparable', () => {
-    // Values without a resume token should be comparable
-    expect(
-      new MongoLSN({
-        timestamp: mongo.Timestamp.fromNumber(1)
-      }).comparable <
-        new MongoLSN({
-          timestamp: mongo.Timestamp.fromNumber(10)
-        }).comparable
-    ).true;
-
-    // Values with resume tokens should correctly compare
-    expect(
-      new MongoLSN({
-        timestamp: mongo.Timestamp.fromNumber(1),
-        resume_token: { _data: 'resume1' }
-      }).comparable <
-        new MongoLSN({
-          timestamp: mongo.Timestamp.fromNumber(10),
-          resume_token: { _data: 'resume2' }
-        }).comparable
-    ).true;
-
-    // The resume token should not affect comparison
-    expect(
-      new MongoLSN({
-        timestamp: mongo.Timestamp.fromNumber(1),
-        resume_token: { _data: '2' }
-      }).comparable <
-        new MongoLSN({
-          timestamp: mongo.Timestamp.fromNumber(10),
-          resume_token: { _data: '1' }
-        }).comparable
-    ).true;
-
-    // Resume token should not be required for comparison
-    expect(
-      new MongoLSN({
-        timestamp: mongo.Timestamp.fromNumber(10),
-        resume_token: { _data: '2' }
-      }).comparable > // Switching the order to test this case
-        new MongoLSN({
-          timestamp: mongo.Timestamp.fromNumber(9)
-        }).comparable
-    ).true;
+import { describeWithStorage } from './util.js';
 
-    // Comparison should be backwards compatible with old LSNs
-    expect(
-      new MongoLSN({
-        timestamp: mongo.Timestamp.fromNumber(10),
-        resume_token: { _data: '2' }
-      }).comparable > ZERO_LSN
-    ).true;
-    expect(
-      new MongoLSN({
-        timestamp: mongo.Timestamp.fromNumber(10),
-        resume_token: { _data: '2' }
-      }).comparable >
-        new MongoLSN({
-          timestamp: mongo.Timestamp.fromNumber(1)
-        }).comparable.split('|')[0] // Simulate an old LSN
-    ).true;
-    expect(
-      new MongoLSN({
-        timestamp: mongo.Timestamp.fromNumber(1),
-        resume_token: { _data: '2' }
-      }).comparable <
-        new MongoLSN({
-          timestamp: mongo.Timestamp.fromNumber(10)
-        }).comparable.split('|')[0] // Simulate an old LSN
-    ).true;
-  });
-});
-
-describe.skipIf(!env.TEST_MONGO_STORAGE)('MongoDB resume - mongo storage', () => {
-  defineResumeTest(INITIALIZED_MONGO_STORAGE_FACTORY);
-});
-
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)('MongoDB resume - postgres storage', () => {
-  defineResumeTest(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+describe('mongodb resuming replication', () => {
+  describeWithStorage({}, defineResumeTest);
 });
 
 function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Promise<BucketStorageFactory>) {
@@ -112,13 +30,7 @@ function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Pr
     await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
 
     // Wait for the item above to be replicated. The commit should store a resume token.
-    await vi.waitFor(
-      async () => {
-        const checkpoint = await context.storage?.getCheckpoint();
-        expect(MongoLSN.fromSerialized(checkpoint!.lsn!).resumeToken).exist;
-      },
-      { timeout: 5000 }
-    );
+    await context.getCheckpoint();
 
     // Done with this context for now
     await context.dispose();
@@ -145,6 +57,7 @@ function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Pr
     const activeContent = await factory.getActiveSyncRulesContent();
     context2.storage = factory.getInstance(activeContent!);
 
+    // If this test times out, it likely didn't throw the expected error here.
     const error = await context2.startStreaming().catch((ex) => ex);
     // The ChangeStreamReplicationJob will detect this and throw a ChangeStreamInvalidatedError
     expect(error).toBeInstanceOf(ChangeStreamInvalidatedError);

package/test/src/resume_token.test.ts

@@ -1,5 +1,6 @@
-import { parseResumeTokenTimestamp } from '@module/common/MongoLSN.js';
-import { describe, expect, it } from 'vitest';
+import { MongoLSN, parseResumeTokenTimestamp, ZERO_LSN } from '@module/common/MongoLSN.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { describe, expect, it, test } from 'vitest';
 
 describe('parseResumeTokenTimestamp', () => {
   it('parses a valid resume token (1)', () => {
@@ -33,3 +34,78 @@ describe('parseResumeTokenTimestamp', () => {
     expect(() => parseResumeTokenTimestamp(resumeToken)).toThrowError(/^Invalid resume token/);
   });
 });
+
+describe('mongo lsn', () => {
+  test('LSN with resume tokens should be comparable', () => {
+    // Values without a resume token should be comparable
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1)
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10)
+        }).comparable
+    ).toBe(true);
+
+    // Values with resume tokens should correctly compare
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1),
+        resume_token: { _data: 'resume1' }
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10),
+          resume_token: { _data: 'resume2' }
+        }).comparable
+    ).toBe(true);
+
+    // The resume token should not affect comparison
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1),
+        resume_token: { _data: '2' }
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10),
+          resume_token: { _data: '1' }
+        }).comparable
+    ).toBe(true);
+
+    // Resume token should not be required for comparison
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(10),
+        resume_token: { _data: '2' }
+      }).comparable > // Switching the order to test this case
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(9)
+        }).comparable
+    ).toBe(true);
+
+    // Comparison should be backwards compatible with old LSNs
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(10),
+        resume_token: { _data: '2' }
+      }).comparable > ZERO_LSN
+    ).toBe(true);
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(10),
+        resume_token: { _data: '2' }
+      }).comparable >
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(1)
+        }).comparable.split('|')[0] // Simulate an old LSN
+    ).toBe(true);
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1),
+        resume_token: { _data: '2' }
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10)
+        }).comparable.split('|')[0] // Simulate an old LSN
+    ).toBe(true);
+  });
+});

package/test/src/resuming_snapshots.test.ts

@@ -0,0 +1,138 @@
+import { describe, expect, test } from 'vitest';
+import { env } from './env.js';
+import { describeWithStorage } from './util.js';
+import { TestStorageFactory } from '@powersync/service-core';
+import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { ReplicationMetric } from '@powersync/service-types';
+import * as timers from 'node:timers/promises';
+import { ChangeStreamTestContext } from './change_stream_utils.js';
+
+describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
+  describeWithStorage({ timeout: 240_000 }, function (factory) {
+    test('resuming initial replication (1)', async () => {
+      // Stop early - likely to not include deleted row in first replication attempt.
+      await testResumingReplication(factory, 2000);
+    });
+    test('resuming initial replication (2)', async () => {
+      // Stop late - likely to include deleted row in first replication attempt.
+      await testResumingReplication(factory, 8000);
+    });
+  });
+});
+
+async function testResumingReplication(factory: TestStorageFactory, stopAfter: number) {
+  // This tests interrupting and then resuming initial replication.
+  // We interrupt replication after test_data1 has fully replicated, and
+  // test_data2 has partially replicated.
+  // This test relies on interval behavior that is not 100% deterministic:
+  // 1. We attempt to abort initial replication once a certain number of
+  //    rows have been replicated, but this is not exact. Our only requirement
+  //    is that we have not fully replicated test_data2 yet.
+  // 2. Order of replication is not deterministic, so which specific rows
+  //    have been / have not been replicated at that point is not deterministic.
+  //    We do allow for some variation in the test results to account for this.
+
+  await using context = await ChangeStreamTestContext.open(factory, { streamOptions: { snapshotChunkLength: 1000 } });
+
+  await context.updateSyncRules(`bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM test_data1
+      - SELECT _id as id, description FROM test_data2`);
+  const { db } = context;
+
+  let batch = db.collection('test_data1').initializeUnorderedBulkOp();
+  for (let i = 1; i <= 1000; i++) {
+    batch.insert({ _id: i, description: 'foo' });
+  }
+  await batch.execute();
+  batch = db.collection('test_data2').initializeUnorderedBulkOp();
+  for (let i = 1; i <= 10000; i++) {
+    batch.insert({ _id: i, description: 'foo' });
+  }
+  await batch.execute();
+
+  const p = context.replicateSnapshot();
+
+  let done = false;
+
+  const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+  try {
+    (async () => {
+      while (!done) {
+        const count =
+          ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
+
+        if (count >= stopAfter) {
+          break;
+        }
+        await timers.setTimeout(1);
+      }
+      // This interrupts initial replication
+      await context.dispose();
+    })();
+    // This confirms that initial replication was interrupted
+    await expect(p).rejects.toThrowError();
+    done = true;
+  } finally {
+    done = true;
+  }
+
+  // Bypass the usual "clear db on factory open" step.
+  await using context2 = await ChangeStreamTestContext.open(factory, {
+    doNotClear: true,
+    streamOptions: { snapshotChunkLength: 1000 }
+  });
+
+  // This delete should be using one of the ids already replicated
+  await db.collection('test_data2').deleteOne({ _id: 1 as any });
+  await db.collection('test_data2').updateOne({ _id: 2 as any }, { $set: { description: 'update1' } });
+  await db.collection('test_data2').insertOne({ _id: 10001 as any, description: 'insert1' });
+
+  await context2.loadNextSyncRules();
+  await context2.replicateSnapshot();
+
+  context2.startStreaming();
+  const data = await context2.getBucketData('global[]', undefined, {});
+
+  const deletedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === '1');
+  const updatedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === '2');
+  const insertedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === '10001');
+
+  if (deletedRowOps.length != 0) {
+    // The deleted row was part of the first replication batch,
+    // so it is removed by streaming replication.
+    expect(deletedRowOps.length).toEqual(2);
+    expect(deletedRowOps[1].op).toEqual('REMOVE');
+  } else {
+    // The deleted row was not part of the first replication batch,
+    // so it's not in the resulting ops at all.
+  }
+
+  expect(updatedRowOps.length).toEqual(2);
+  // description for the first op could be 'foo' or 'update1'.
+  // We only test the final version.
+  expect(JSON.parse(updatedRowOps[1].data as string).description).toEqual('update1');
+
+  expect(insertedRowOps.length).toEqual(2);
+  expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1');
+  expect(JSON.parse(insertedRowOps[1].data as string).description).toEqual('insert1');
+
+  // 1000 of test_data1 during first replication attempt.
+  // N >= 1000 of test_data2 during first replication attempt.
+  // 10000 - N - 1 + 1 of test_data2 during second replication attempt.
+  // An additional update during streaming replication (2x total for this row).
+  // An additional insert during streaming replication (2x total for this row).
+  // If the deleted row was part of the first replication batch, it's removed by streaming replication.
+  // This adds 2 ops.
+  // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000.
+  // However, this is not deterministic.
+  const expectedCount = 11002 + deletedRowOps.length;
+  expect(data.length).toEqual(expectedCount);
+
+  const replicatedCount =
+    ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
+
+  // With resumable replication, there should be no need to re-replicate anything.
+  expect(replicatedCount).toEqual(expectedCount);
+}

package/test/src/slow_tests.test.ts

@@ -6,24 +6,10 @@ import { storage } from '@powersync/service-core';
 
 import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js';
 import { env } from './env.js';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
-
-describe.skipIf(!env.TEST_MONGO_STORAGE)('change stream slow tests - mongodb', { timeout: 60_000 }, function () {
-  if (env.CI || env.SLOW_TESTS) {
-    defineSlowTests(INITIALIZED_MONGO_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
-});
+import { describeWithStorage } from './util.js';
 
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)('change stream slow tests - postgres', { timeout: 60_000 }, function () {
-  if (env.CI || env.SLOW_TESTS) {
-    defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
+describe.runIf(env.CI || env.SLOW_TESTS)('change stream slow tests', { timeout: 60_000 }, function () {
+  describeWithStorage({}, defineSlowTests);
 });
 
 function defineSlowTests(factory: storage.TestStorageFactory) {
@@ -96,7 +82,7 @@ bucket_definitions:
     await snapshotPromise;
     context.startStreaming();
 
-    const data = await context.getBucketData('global[]', undefined, { limit: 50_000, chunkLimitBytes: 60_000_000 });
+    const data = await context.getBucketData('global[]');
 
     const preDocuments = data.filter((d: any) => JSON.parse(d.data! as string).description.startsWith('pre')).length;
     const updatedDocuments = data.filter((d: any) =>

package/test/src/util.ts

@@ -4,7 +4,8 @@ import * as postgres_storage from '@powersync/service-module-postgres-storage';
 
 import * as types from '@module/types/types.js';
 import { env } from './env.js';
-import { BSON_DESERIALIZE_DATA_OPTIONS } from '@powersync/service-core';
+import { BSON_DESERIALIZE_DATA_OPTIONS, TestStorageFactory } from '@powersync/service-core';
+import { describe, TestOptions } from 'vitest';
 
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
 
@@ -22,6 +23,16 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTes
   url: env.PG_STORAGE_TEST_URL
 });
 
+export function describeWithStorage(options: TestOptions, fn: (factory: TestStorageFactory) => void) {
+  describe.skipIf(!env.TEST_MONGO_STORAGE)(`mongodb storage`, options, function () {
+    fn(INITIALIZED_MONGO_STORAGE_FACTORY);
+  });
+
+  describe.skipIf(!env.TEST_POSTGRES_STORAGE)(`postgres storage`, options, function () {
+    fn(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+  });
+}
+
 export async function clearTestDb(db: mongo.Db) {
   await db.dropDatabase();
 }