@powersync/service-module-mongodb-storage 0.0.0-dev-20250108073049

package/test/src/storage.test.ts

@@ -0,0 +1,7 @@
+import { register } from '@powersync/service-core-tests';
+import { describe } from 'vitest';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+
+describe('Mongo Sync Bucket Storage', () => register.registerDataStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+
+describe('Sync Bucket Validation', register.registerBucketValidationTests);

package/test/src/storage_compacting.test.ts

@@ -0,0 +1,6 @@
+import { MongoCompactOptions } from '@module/storage/implementation/MongoCompactor.js';
+import { register } from '@powersync/service-core-tests';
+import { describe } from 'vitest';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+
+describe('Mongo Sync Bucket Storage Compact', () => register.registerCompactTests<MongoCompactOptions>(INITIALIZED_MONGO_STORAGE_FACTORY, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));

package/test/src/storage_sync.test.ts

@@ -0,0 +1,113 @@
+import { storage } from '@powersync/service-core';
+import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+
+describe('sync - mongodb', () => {
+  register.registerSyncTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+
+  // The split of returned results can vary depending on storage drivers
+  test('large batch (2)', async () => {
+    // Test syncing a batch of data that is small in count,
+    // but large enough in size to be split over multiple returned chunks.
+    // Similar to the above test, but splits over 1MB chunks.
+    const sync_rules = test_utils.testRules(
+      `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM "%"
+    `
+    );
+    using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+    const bucketStorage = factory.getInstance(sync_rules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      const sourceTable = TEST_TABLE;
+
+      const largeDescription = '0123456789'.repeat(2_000_00);
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test1',
+          description: 'test1'
+        },
+        afterReplicaId: test_utils.rid('test1')
+      });
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large1',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large1')
+      });
+
+      // Large enough to split the returned batch
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large2',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large2')
+      });
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test3',
+          description: 'test3'
+        },
+        afterReplicaId: test_utils.rid('test3')
+      });
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    const options: storage.BucketDataBatchOptions = {};
+
+    const batch1 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options)
+    );
+    expect(test_utils.getBatchData(batch1)).toEqual([
+      { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
+      { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+    ]);
+    expect(test_utils.getBatchMeta(batch1)).toEqual({
+      after: '0',
+      has_more: true,
+      next_after: '2'
+    });
+
+    const batch2 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options)
+    );
+    expect(test_utils.getBatchData(batch2)).toEqual([
+      { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
+    ]);
+    expect(test_utils.getBatchMeta(batch2)).toEqual({
+      after: '2',
+      has_more: true,
+      next_after: '3'
+    });
+
+    const batch3 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options)
+    );
+    expect(test_utils.getBatchData(batch3)).toEqual([
+      { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+    ]);
+    expect(test_utils.getBatchMeta(batch3)).toEqual({
+      after: '3',
+      has_more: false,
+      next_after: '4'
+    });
+  });
+});

package/test/src/util.ts

@@ -0,0 +1,8 @@
+import { env } from './env.js';
+
+import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js';
+
+export const INITIALIZED_MONGO_STORAGE_FACTORY = MongoTestStorageFactoryGenerator({
+  url: env.MONGO_TEST_URL,
+  isCI: env.CI
+});

package/test/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "baseUrl": "./",
+    "noEmit": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "paths": {
+      "@/*": ["../../../packages/service-core/src/*"],
+      "@module/*": ["../src/*"],
+      "@core-tests/*": ["../../../packages/service-core/test/src/*"]
+    }
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../"
+    },
+    {
+      "path": "../../../packages/service-core/test"
+    },
+    {
+      "path": "../../../packages/service-core/"
+    },
+    {
+      "path": "../../../packages/service-core-tests/"
+    }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../../packages/types"
+    },
+    {
+      "path": "../../packages/jsonbig"
+    },
+    {
+      "path": "../../packages/sync-rules"
+    },
+    {
+      "path": "../../packages/service-core"
+    },
+    {
+      "path": "../../libs/lib-services"
+    },
+    {
+      "path": "../../libs/lib-mongodb"
+    }
+  ]
+}