@powersync/service-module-mongodb 0.0.0-dev-20250102111825 → 0.0.0-dev-20250108084515

package/test/src/change_stream.test.ts

@@ -1,7 +1,7 @@
 import { test_utils } from '@powersync/service-core-tests';
 
 import { PostImagesOption } from '@module/types/types.js';
-import { BucketStorageFactory } from '@powersync/service-core';
+import { storage } from '@powersync/service-core';
 import * as crypto from 'crypto';
 import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
@@ -9,8 +9,6 @@ import { describe, expect, test, vi } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
-type StorageFactory = () => Promise<BucketStorageFactory>;
-
 const BASIC_SYNC_RULES = `
 bucket_definitions:
   global:
@@ -22,7 +20,7 @@ describe('change stream - mongodb', { timeout: 20_000 }, function () {
   defineChangeStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 });
 
-function defineChangeStreamTests(factory: StorageFactory) {
+function defineChangeStreamTests(factory: storage.TestStorageFactory) {
   test('replicating basic values', async () => {
     await using context = await ChangeStreamTestContext.open(factory);
     const { db } = context;

package/test/src/env.ts

@@ -1,8 +1,8 @@
 import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
-  MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'),
   MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
+  MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'),
   CI: utils.type.boolean.default('false'),
   SLOW_TESTS: utils.type.boolean.default('false')
 });

package/test/src/mongo_test.test.ts

@@ -45,12 +45,52 @@ describe('mongo data types', () => {
           mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'),
           'mydb',
           { foo: 'bar' }
-        ),
-        undefined: undefined
+        )
       }
     ]);
   }
 
+  async function insertUndefined(db: mongo.Db, collection: string, array?: boolean) {
+    // MongoDB has deprecated the `undefined` value, making it really
+    // difficult to insert one into the database.
+    // mapReduce is also deprecated, but it's one way to still generate
+    // the value.
+    const mapInput = db.collection('map_input');
+    await mapInput.insertOne({ test: 'test' });
+    const fin = array ? `return { result: [undefined] }` : `return { result: undefined }`;
+    await db.command({
+      mapReduce: 'map_input',
+      map: new mongo.Code(`function () {
+          // We only need to emit once for a single result:
+          emit(5, {});
+        }`),
+      reduce: new mongo.Code(`function (key, values) {
+          // Return an object whose property is explicitly set to undefined
+          return undefined;
+        }`),
+      finalize: new mongo.Code(`function (key, reducedVal) {
+          ${fin};
+        }`),
+      out: { merge: 'map_output' }
+    });
+
+    await db
+      .collection('map_output')
+      .aggregate([
+        { $set: { undefined: '$value.result' } },
+        { $project: { undefined: 1 } },
+        {
+          $merge: {
+            into: collection
+          }
+        }
+      ])
+      .toArray();
+
+    await mapInput.drop();
+    await db.collection('map_output').drop();
+  }
+
   async function insertNested(collection: mongo.Collection) {
     await collection.insertMany([
       {
@@ -118,9 +158,11 @@ describe('mongo data types', () => {
       js: '{"code":"testcode","scope":null}',
       js2: '{"code":"testcode","scope":{"foo":"bar"}}',
       pointer: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}',
-      pointer2: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}',
-      undefined: null
+      pointer2: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}'
     });
+
+    // This must specifically be null, and not undefined.
+    expect(transformed[4].undefined).toBeNull();
   }
 
   function checkResultsNested(transformed: Record<string, any>[]) {
@@ -158,20 +200,27 @@ describe('mongo data types', () => {
       js: '[{"code":"testcode","scope":null}]',
       pointer: '[{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}]',
       minKey: '[null]',
-      maxKey: '[null]',
+      maxKey: '[null]'
+    });
+
+    expect(transformed[4]).toMatchObject({
       undefined: '[null]'
     });
   }
 
   test('test direct queries', async () => {
     const { db, client } = await connectMongoData();
+
     const collection = db.collection('test_data');
     try {
       await setupTable(db);
-
       await insert(collection);
+      await insertUndefined(db, 'test_data');
 
-      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data').find().toArray())];
+      const rawResults = await db.collection('test_data').find().toArray();
+      // It is tricky to save "undefined" with mongo, so we check that it succeeded.
+      expect(rawResults[4].undefined).toBeUndefined();
+      const transformed = [...ChangeStream.getQueryData(rawResults)];
 
       checkResults(transformed);
     } finally {
@@ -186,8 +235,11 @@ describe('mongo data types', () => {
       await setupTable(db);
 
       await insertNested(collection);
+      await insertUndefined(db, 'test_data_arrays', true);
 
-      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data_arrays').find().toArray())];
+      const rawResults = await db.collection('test_data_arrays').find().toArray();
+      expect(rawResults[4].undefined).toEqual([undefined]);
+      const transformed = [...ChangeStream.getQueryData(rawResults)];
 
       checkResultsNested(transformed);
     } finally {
@@ -212,8 +264,9 @@ describe('mongo data types', () => {
       await stream.tryNext();
 
       await insert(collection);
+      await insertUndefined(db, 'test_data');
 
-      const transformed = await getReplicationTx(stream, 4);
+      const transformed = await getReplicationTx(stream, 5);
 
       checkResults(transformed);
     } finally {
@@ -236,8 +289,9 @@ describe('mongo data types', () => {
       await stream.tryNext();
 
       await insertNested(collection);
+      await insertUndefined(db, 'test_data_arrays', true);
 
-      const transformed = await getReplicationTx(stream, 4);
+      const transformed = await getReplicationTx(stream, 5);
 
       checkResultsNested(transformed);
     } finally {
@@ -256,6 +310,7 @@ describe('mongo data types', () => {
     const collection = db.collection('test_data');
     await setupTable(db);
     await insert(collection);
+    await insertUndefined(db, 'test_data');
 
     const schema = await adapter.getConnectionSchema();
     const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0];
@@ -440,6 +495,10 @@ bucket_definitions:
 async function getReplicationTx(replicationStream: mongo.ChangeStream, count: number) {
   let transformed: SqliteRow[] = [];
   for await (const doc of replicationStream) {
+    // Specifically filter out map_input / map_output collections
+    if (!(doc as any)?.ns?.coll?.startsWith('test_data')) {
+      continue;
+    }
     transformed.push(constructAfterRecord((doc as any).fullDocument));
     if (transformed.length == count) {
       break;

package/test/src/setup.ts

@@ -1,7 +1,10 @@
 import { container } from '@powersync/lib-services-framework';
+import { test_utils } from '@powersync/service-core-tests';
 import { beforeAll } from 'vitest';
 
-beforeAll(() => {
+beforeAll(async () => {
   // Executes for every test file
   container.registerDefaults();
+
+  await test_utils.initMetrics();
 });

package/test/src/slow_tests.test.ts

@@ -1,4 +1,4 @@
-import { BucketStorageFactory } from '@powersync/service-core';
+import { storage } from '@powersync/service-core';
 import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
@@ -6,15 +6,6 @@ import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_str
 import { env } from './env.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
-type StorageFactory = () => Promise<BucketStorageFactory>;
-
-const BASIC_SYNC_RULES = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT _id as id, description FROM "test_data"
-`;
-
 describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () {
   if (env.CI || env.SLOW_TESTS) {
     defineSlowTests(INITIALIZED_MONGO_STORAGE_FACTORY);
@@ -24,7 +15,7 @@ describe('change stream slow tests - mongodb', { timeout: 60_000 }, function ()
   }
 });
 
-function defineSlowTests(factory: StorageFactory) {
+function defineSlowTests(factory: storage.TestStorageFactory) {
   test('replicating snapshot with lots of data', async () => {
     await using context = await ChangeStreamTestContext.open(factory);
     // Test with low minSnapshotHistoryWindowInSeconds, to trigger:
@@ -97,7 +88,9 @@ bucket_definitions:
     const data = await context.getBucketData('global[]', undefined, { limit: 50_000, chunkLimitBytes: 60_000_000 });
 
     const preDocuments = data.filter((d: any) => JSON.parse(d.data! as string).description.startsWith('pre')).length;
-    const updatedDocuments = data.filter((d: any) => JSON.parse(d.data! as string).description.startsWith('updated')).length;
+    const updatedDocuments = data.filter((d: any) =>
+      JSON.parse(d.data! as string).description.startsWith('updated')
+    ).length;
 
     // If the test works properly, preDocuments should be around 2000-3000.
     // The total should be around 9000-9900.

package/test/src/util.ts

@@ -1,20 +1,9 @@
 import * as types from '@module/types/types.js';
-import { BucketStorageFactory, Metrics } from '@powersync/service-core';
 
-import { MongoBucketStorage } from '@module/storage/MongoBucketStorage.js';
-import { PowerSyncMongo } from '@module/storage/storage-index.js';
-import { test_utils } from '@powersync/service-core-tests';
+import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as mongo from 'mongodb';
 import { env } from './env.js';
 
-// The metrics need to be initialized before they can be used
-await Metrics.initialise({
-  disable_telemetry_sharing: true,
-  powersync_instance_id: 'test',
-  internal_metrics_endpoint: 'unused.for.tests.com'
-});
-Metrics.getInstance().resetCounters();
-
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
 
 export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
@@ -22,21 +11,10 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
   uri: TEST_URI
 });
 
-export type StorageFactory = () => Promise<BucketStorageFactory>;
-
-export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async (options?: test_utils.StorageOptions) => {
-  const db = await connectMongo();
-
-  // None of the PG tests insert data into this collection, so it was never created
-  if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-    await db.db.createCollection('bucket_parameters');
-  }
-  if (!options?.doNotClear) {
-    await db.clear();
-  }
-
-  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
-};
+export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageFactoryGenerator({
+  url: env.MONGO_TEST_URL,
+  isCI: env.CI
+});
 
 export async function clearTestDb(db: mongo.Db) {
   await db.dropDatabase();
@@ -52,14 +30,3 @@ export async function connectMongoData() {
   const dbname = new URL(env.MONGO_TEST_DATA_URL).pathname.substring(1);
   return { client, db: client.db(dbname) };
 }
-
-export async function connectMongo() {
-  // Short timeout for tests, to fail fast when the server is not available.
-  // Slightly longer timeouts for CI, to avoid arbitrary test failures
-  const client = new mongo.MongoClient(env.MONGO_TEST_URL, {
-    connectTimeoutMS: env.CI ? 15_000 : 5_000,
-    socketTimeoutMS: env.CI ? 15_000 : 5_000,
-    serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500
-  });
-  return new PowerSyncMongo(client);
-}

package/test/tsconfig.json

@@ -23,8 +23,7 @@
     },
     {
       "path": "../../../packages/service-core/"
-    }
-    ,
+    },
     {
       "path": "../../../packages/service-core-tests/"
     }

package/tsconfig.json

@@ -23,6 +23,12 @@
     },
     {
       "path": "../../libs/lib-services"
+    },
+    {
+      "path": "../../libs/lib-mongodb"
+    },
+    {
+      "path": "../module-mongodb-storage"
     }
   ]
 }