@powersync/service-module-mongodb 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219110735

package/test/src/storage_sync.test.ts

@@ -0,0 +1,113 @@
+import { storage } from '@powersync/service-core';
+import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+
+describe('sync - mongodb', () => {
+  register.registerSyncTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+
+  // The split of returned results can vary depending on storage drivers
+  test('large batch (2)', async () => {
+    // Test syncing a batch of data that is small in count,
+    // but large enough in size to be split over multiple returned chunks.
+    // Similar to the above test, but splits over 1MB chunks.
+    const sync_rules = test_utils.testRules(
+      `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM "%"
+    `
+    );
+    using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+    const bucketStorage = factory.getInstance(sync_rules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      const sourceTable = TEST_TABLE;
+
+      const largeDescription = '0123456789'.repeat(2_000_00);
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test1',
+          description: 'test1'
+        },
+        afterReplicaId: test_utils.rid('test1')
+      });
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large1',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large1')
+      });
+
+      // Large enough to split the returned batch
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large2',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large2')
+      });
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test3',
+          description: 'test3'
+        },
+        afterReplicaId: test_utils.rid('test3')
+      });
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    const options: storage.BucketDataBatchOptions = {};
+
+    const batch1 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options)
+    );
+    expect(test_utils.getBatchData(batch1)).toEqual([
+      { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
+      { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+    ]);
+    expect(test_utils.getBatchMeta(batch1)).toEqual({
+      after: '0',
+      has_more: true,
+      next_after: '2'
+    });
+
+    const batch2 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options)
+    );
+    expect(test_utils.getBatchData(batch2)).toEqual([
+      { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
+    ]);
+    expect(test_utils.getBatchMeta(batch2)).toEqual({
+      after: '2',
+      has_more: true,
+      next_after: '3'
+    });
+
+    const batch3 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options)
+    );
+    expect(test_utils.getBatchData(batch3)).toEqual([
+      { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+    ]);
+    expect(test_utils.getBatchMeta(batch3)).toEqual({
+      after: '3',
+      has_more: false,
+      next_after: '4'
+    });
+  });
+});

package/test/src/util.ts

@@ -1,10 +1,11 @@
 import * as types from '@module/types/types.js';
-import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId } from '@powersync/service-core';
+import { BucketStorageFactory, Metrics } from '@powersync/service-core';
 
-import { env } from './env.js';
-import { logger } from '@powersync/lib-services-framework';
-import { connectMongo } from '@core-tests/util.js';
+import { MongoBucketStorage } from '@module/storage/MongoBucketStorage.js';
+import { PowerSyncMongo } from '@module/storage/storage-index.js';
+import { test_utils } from '@powersync/service-core-tests';
 import * as mongo from 'mongodb';
+import { env } from './env.js';
 
 // The metrics need to be initialized before they can be used
 await Metrics.initialise({
@@ -23,15 +24,16 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
 
 export type StorageFactory = () => Promise<BucketStorageFactory>;
 
-export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => {
+export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async (options?: test_utils.StorageOptions) => {
   const db = await connectMongo();
 
   // None of the PG tests insert data into this collection, so it was never created
   if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
     await db.db.createCollection('bucket_parameters');
   }
-
-  await db.clear();
+  if (!options?.doNotClear) {
+    await db.clear();
+  }
 
   return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
 };
@@ -50,3 +52,14 @@ export async function connectMongoData() {
   const dbname = new URL(env.MONGO_TEST_DATA_URL).pathname.substring(1);
   return { client, db: client.db(dbname) };
 }
+
+export async function connectMongo() {
+  // Short timeout for tests, to fail fast when the server is not available.
+  // Slightly longer timeouts for CI, to avoid arbitrary test failures
+  const client = new mongo.MongoClient(env.MONGO_TEST_URL, {
+    connectTimeoutMS: env.CI ? 15_000 : 5_000,
+    socketTimeoutMS: env.CI ? 15_000 : 5_000,
+    serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500
+  });
+  return new PowerSyncMongo(client);
+}

package/test/tsconfig.json

@@ -24,5 +24,9 @@
     {
       "path": "../../../packages/service-core/"
     }
+    ,
+    {
+      "path": "../../../packages/service-core-tests/"
+    }
   ]
 }