@powersync/service-module-mongodb 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219091224

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -0,0 +1,332 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
+[
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": -93886621n,
+          "op": "CLEAR",
+          "op_id": "2",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "4",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 499012468,
+          "count": 4,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "2",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 1859363232n,
+          "data": "{"id":"t1","description":"Test 1b"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "3",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+        {
+          "checksum": 3028503153n,
+          "data": "{"id":"t2","description":"Test 2b"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "4",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
+        },
+      ],
+      "has_more": false,
+      "next_after": "4",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "4",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > expired token 1`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - mongodb > expiring token 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 0,
+          "count": 0,
+        },
+      ],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > expiring token 2`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - mongodb > sync global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+        {
+          "checksum": 3280762209n,
+          "data": "{"id":"t2","description":"Test 2"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync legacy non-raw data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -852817836,
+          "count": 1,
+        },
+      ],
+      "last_op_id": "1",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 3442149460n,
+          "data": {
+            "description": "Test
+"string"",
+            "id": "t1",
+            "large_num": 12345678901234567890n,
+          },
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 0,
+          "count": 0,
+        },
+      ],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to global data 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 920318466,
+          "count": 1,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to global data 3`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "1",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 3280762209n,
+          "data": "{"id":"t2","description":"Test 2"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;

package/test/src/change_stream.test.ts

@@ -1,12 +1,13 @@
-import { putOp, removeOp } from '@core-tests/stream_utils.js';
-import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
+import { test_utils } from '@powersync/service-core-tests';
+
+import { PostImagesOption } from '@module/types/types.js';
 import { BucketStorageFactory } from '@powersync/service-core';
 import * as crypto from 'crypto';
 import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test, vi } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
-import { PostImagesOption } from '@module/types/types.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 type StorageFactory = () => Promise<BucketStorageFactory>;
 
@@ -18,7 +19,7 @@ bucket_definitions:
 `;
 
 describe('change stream - mongodb', { timeout: 20_000 }, function () {
-  defineChangeStreamTests(MONGO_STORAGE_FACTORY);
+  defineChangeStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 });
 
 function defineChangeStreamTests(factory: StorageFactory) {
@@ -52,10 +53,10 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id.toHexString())
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id.toHexString())
     ]);
   });
 
@@ -86,8 +87,8 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n })
     ]);
   });
 
@@ -125,11 +126,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // fullDocument is not available at the point this is replicated, resulting in it treated as a remove
-      removeOp('test_data', test_id!.toHexString()),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.removeOp('test_data', test_id!.toHexString()),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -171,11 +172,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // The postImage helps us get this data
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -216,11 +217,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // The postImage helps us get this data
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -244,7 +245,7 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
 
-    expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
+    expect(data).toMatchObject([test_utils.putOp('test_DATA', { id: test_id, description: 'test1' })]);
   });
 
   test('replicating large values', async () => {
@@ -270,10 +271,10 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
     expect(data.slice(0, 1)).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription })
     ]);
     expect(data.slice(1)).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription })
     ]);
   });
 
@@ -302,8 +303,8 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id, description: 'test1' }),
-      removeOp('test_data', test_id)
+      test_utils.putOp('test_data', { id: test_id, description: 'test1' }),
+      test_utils.removeOp('test_data', test_id)
     ]);
   });
 
@@ -330,9 +331,9 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data1', { id: test_id, description: 'test1' }),
-      removeOp('test_data1', test_id),
-      putOp('test_data2', { id: test_id, description: 'test1' })
+      test_utils.putOp('test_data1', { id: test_id, description: 'test1' }),
+      test_utils.removeOp('test_data1', test_id),
+      test_utils.putOp('test_data2', { id: test_id, description: 'test1' })
     ]);
   });
 
@@ -349,7 +350,7 @@ bucket_definitions:
     context.startStreaming();
 
     const data = await context.getBucketData('global[]');
-    expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
+    expect(data).toMatchObject([test_utils.putOp('test_data', { id: test_id, description: 'test1' })]);
   });
 
   test('large record', async () => {
@@ -446,8 +447,8 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
     ]);
   });
 

package/test/src/change_stream_utils.ts

@@ -1,12 +1,12 @@
 import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core';
 
-import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
-import { fromAsync } from '@core-tests/stream_utils.js';
-import { MongoManager } from '@module/replication/MongoManager.js';
 import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
-import * as mongo from 'mongodb';
+import { MongoManager } from '@module/replication/MongoManager.js';
 import { createCheckpoint } from '@module/replication/MongoRelation.js';
 import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
+import { test_utils } from '@powersync/service-core-tests';
+import * as mongo from 'mongodb';
+import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
 
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
@@ -102,7 +102,7 @@ export class ChangeStreamTestContext {
   async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
     const map = new Map<string, string>(Object.entries(buckets));
-    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+    return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
   async getBucketData(
@@ -117,7 +117,7 @@ export class ChangeStreamTestContext {
       limit: options?.limit,
       chunkLimitBytes: options?.chunkLimitBytes
     });
-    const batches = await fromAsync(batch);
+    const batches = await test_utils.fromAsync(batch);
     return batches[0]?.batch.data ?? [];
   }
 

package/test/src/env.ts

@@ -2,6 +2,7 @@ import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
   MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'),
+  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   CI: utils.type.boolean.default('false'),
   SLOW_TESTS: utils.type.boolean.default('false')
 });

package/test/src/slow_tests.test.ts

@@ -1,10 +1,10 @@
-import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
 import { BucketStorageFactory } from '@powersync/service-core';
 import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
 import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js';
 import { env } from './env.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 type StorageFactory = () => Promise<BucketStorageFactory>;
 
@@ -17,7 +17,7 @@ bucket_definitions:
 
 describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () {
   if (env.CI || env.SLOW_TESTS) {
-    defineSlowTests(MONGO_STORAGE_FACTORY);
+    defineSlowTests(INITIALIZED_MONGO_STORAGE_FACTORY);
   } else {
     // Need something in this file.
     test('no-op', () => {});
@@ -96,8 +96,8 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]', undefined, { limit: 50_000, chunkLimitBytes: 60_000_000 });
 
-    const preDocuments = data.filter((d) => JSON.parse(d.data! as string).description.startsWith('pre')).length;
-    const updatedDocuments = data.filter((d) => JSON.parse(d.data! as string).description.startsWith('updated')).length;
+    const preDocuments = data.filter((d: any) => JSON.parse(d.data! as string).description.startsWith('pre')).length;
+    const updatedDocuments = data.filter((d: any) => JSON.parse(d.data! as string).description.startsWith('updated')).length;
 
     // If the test works properly, preDocuments should be around 2000-3000.
     // The total should be around 9000-9900.

package/test/src/storage.test.ts

@@ -0,0 +1,7 @@
+import { register } from '@powersync/service-core-tests';
+import { describe } from 'vitest';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+
+describe('Mongo Sync Bucket Storage', () => register.registerDataStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+
+describe('Sync Bucket Validation', register.registerBucketValidationTests);

package/test/src/storage_compacting.test.ts

@@ -0,0 +1,6 @@
+import { MongoCompactOptions } from '@module/storage/implementation/MongoCompactor.js';
+import { register } from '@powersync/service-core-tests';
+import { describe } from 'vitest';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+
+describe('Mongo Sync Bucket Storage Compact', () => register.registerCompactTests<MongoCompactOptions>(INITIALIZED_MONGO_STORAGE_FACTORY, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));