@powersync/service-module-mongodb 0.1.8 → 0.2.0

package/test/src/change_stream.test.ts

@@ -1,14 +1,13 @@
-import { putOp, removeOp } from '@core-tests/stream_utils.js';
-import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
-import { BucketStorageFactory } from '@powersync/service-core';
+import { test_utils } from '@powersync/service-core-tests';
+
+import { PostImagesOption } from '@module/types/types.js';
+import { storage } from '@powersync/service-core';
 import * as crypto from 'crypto';
 import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test, vi } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
-import { PostImagesOption } from '@module/types/types.js';
-
-type StorageFactory = () => Promise<BucketStorageFactory>;
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:
@@ -18,10 +17,10 @@ bucket_definitions:
 `;
 
 describe('change stream - mongodb', { timeout: 20_000 }, function () {
-  defineChangeStreamTests(MONGO_STORAGE_FACTORY);
+  defineChangeStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 });
 
-function defineChangeStreamTests(factory: StorageFactory) {
+function defineChangeStreamTests(factory: storage.TestStorageFactory) {
   test('replicating basic values', async () => {
     await using context = await ChangeStreamTestContext.open(factory);
     const { db } = context;
@@ -52,10 +51,10 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id.toHexString())
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id.toHexString())
     ]);
   });
 
@@ -86,8 +85,8 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n })
     ]);
   });
 
@@ -125,11 +124,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // fullDocument is not available at the point this is replicated, resulting in it treated as a remove
-      removeOp('test_data', test_id!.toHexString()),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.removeOp('test_data', test_id!.toHexString()),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -171,11 +170,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // The postImage helps us get this data
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -216,11 +215,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // The postImage helps us get this data
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -244,7 +243,7 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
 
-    expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
+    expect(data).toMatchObject([test_utils.putOp('test_DATA', { id: test_id, description: 'test1' })]);
   });
 
   test('replicating large values', async () => {
@@ -270,10 +269,10 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
     expect(data.slice(0, 1)).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription })
     ]);
     expect(data.slice(1)).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription })
     ]);
   });
 
@@ -302,8 +301,8 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id, description: 'test1' }),
-      removeOp('test_data', test_id)
+      test_utils.putOp('test_data', { id: test_id, description: 'test1' }),
+      test_utils.removeOp('test_data', test_id)
     ]);
   });
 
@@ -330,9 +329,9 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data1', { id: test_id, description: 'test1' }),
-      removeOp('test_data1', test_id),
-      putOp('test_data2', { id: test_id, description: 'test1' })
+      test_utils.putOp('test_data1', { id: test_id, description: 'test1' }),
+      test_utils.removeOp('test_data1', test_id),
+      test_utils.putOp('test_data2', { id: test_id, description: 'test1' })
     ]);
   });
 
@@ -349,7 +348,7 @@ bucket_definitions:
     context.startStreaming();
 
     const data = await context.getBucketData('global[]');
-    expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
+    expect(data).toMatchObject([test_utils.putOp('test_data', { id: test_id, description: 'test1' })]);
   });
 
   test('large record', async () => {
@@ -446,8 +445,8 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
     ]);
   });
 

package/test/src/change_stream_utils.ts

@@ -1,12 +1,12 @@
 import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core';
 
-import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
-import { fromAsync } from '@core-tests/stream_utils.js';
-import { MongoManager } from '@module/replication/MongoManager.js';
 import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
-import * as mongo from 'mongodb';
+import { MongoManager } from '@module/replication/MongoManager.js';
 import { createCheckpoint } from '@module/replication/MongoRelation.js';
 import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
+import { test_utils } from '@powersync/service-core-tests';
+import * as mongo from 'mongodb';
+import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
 
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
@@ -102,7 +102,7 @@ export class ChangeStreamTestContext {
   async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
     const map = new Map<string, string>(Object.entries(buckets));
-    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+    return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
   async getBucketData(
@@ -117,7 +117,7 @@ export class ChangeStreamTestContext {
       limit: options?.limit,
       chunkLimitBytes: options?.chunkLimitBytes
     });
-    const batches = await fromAsync(batch);
+    const batches = await test_utils.fromAsync(batch);
     return batches[0]?.batch.data ?? [];
   }
 

package/test/src/env.ts

@@ -1,6 +1,7 @@
 import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
+  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'),
   CI: utils.type.boolean.default('false'),
   SLOW_TESTS: utils.type.boolean.default('false')

package/test/src/mongo_test.test.ts

@@ -45,12 +45,52 @@ describe('mongo data types', () => {
           mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'),
           'mydb',
           { foo: 'bar' }
-        ),
-        undefined: undefined
+        )
       }
     ]);
   }
 
+  async function insertUndefined(db: mongo.Db, collection: string, array?: boolean) {
+    // MongoDB has deprecated the `undefined` value, making it really
+    // difficult to insert one into the database.
+    // mapReduce is also deprecated, but it's one way to still generate
+    // the value.
+    const mapInput = db.collection('map_input');
+    await mapInput.insertOne({ test: 'test' });
+    const fin = array ? `return { result: [undefined] }` : `return { result: undefined }`;
+    await db.command({
+      mapReduce: 'map_input',
+      map: new mongo.Code(`function () {
+          // We only need to emit once for a single result:
+          emit(5, {});
+        }`),
+      reduce: new mongo.Code(`function (key, values) {
+          // Return an object whose property is explicitly set to undefined
+          return undefined;
+        }`),
+      finalize: new mongo.Code(`function (key, reducedVal) {
+          ${fin};
+        }`),
+      out: { merge: 'map_output' }
+    });
+
+    await db
+      .collection('map_output')
+      .aggregate([
+        { $set: { undefined: '$value.result' } },
+        { $project: { undefined: 1 } },
+        {
+          $merge: {
+            into: collection
+          }
+        }
+      ])
+      .toArray();
+
+    await mapInput.drop();
+    await db.collection('map_output').drop();
+  }
+
   async function insertNested(collection: mongo.Collection) {
     await collection.insertMany([
       {
@@ -118,9 +158,11 @@ describe('mongo data types', () => {
       js: '{"code":"testcode","scope":null}',
       js2: '{"code":"testcode","scope":{"foo":"bar"}}',
       pointer: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}',
-      pointer2: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}',
-      undefined: null
+      pointer2: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}'
     });
+
+    // This must specifically be null, and not undefined.
+    expect(transformed[4].undefined).toBeNull();
   }
 
   function checkResultsNested(transformed: Record<string, any>[]) {
@@ -158,20 +200,27 @@ describe('mongo data types', () => {
       js: '[{"code":"testcode","scope":null}]',
       pointer: '[{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}]',
       minKey: '[null]',
-      maxKey: '[null]',
+      maxKey: '[null]'
+    });
+
+    expect(transformed[4]).toMatchObject({
       undefined: '[null]'
     });
   }
 
   test('test direct queries', async () => {
     const { db, client } = await connectMongoData();
+
     const collection = db.collection('test_data');
     try {
       await setupTable(db);
-
       await insert(collection);
+      await insertUndefined(db, 'test_data');
 
-      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data').find().toArray())];
+      const rawResults = await db.collection('test_data').find().toArray();
+      // It is tricky to save "undefined" with mongo, so we check that it succeeded.
+      expect(rawResults[4].undefined).toBeUndefined();
+      const transformed = [...ChangeStream.getQueryData(rawResults)];
 
       checkResults(transformed);
     } finally {
@@ -186,8 +235,11 @@ describe('mongo data types', () => {
       await setupTable(db);
 
       await insertNested(collection);
+      await insertUndefined(db, 'test_data_arrays', true);
 
-      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data_arrays').find().toArray())];
+      const rawResults = await db.collection('test_data_arrays').find().toArray();
+      expect(rawResults[4].undefined).toEqual([undefined]);
+      const transformed = [...ChangeStream.getQueryData(rawResults)];
 
       checkResultsNested(transformed);
     } finally {
@@ -212,8 +264,9 @@ describe('mongo data types', () => {
       await stream.tryNext();
 
       await insert(collection);
+      await insertUndefined(db, 'test_data');
 
-      const transformed = await getReplicationTx(stream, 4);
+      const transformed = await getReplicationTx(stream, 5);
 
       checkResults(transformed);
     } finally {
@@ -236,8 +289,9 @@ describe('mongo data types', () => {
       await stream.tryNext();
 
       await insertNested(collection);
+      await insertUndefined(db, 'test_data_arrays', true);
 
-      const transformed = await getReplicationTx(stream, 4);
+      const transformed = await getReplicationTx(stream, 5);
 
       checkResultsNested(transformed);
     } finally {
@@ -256,6 +310,7 @@ describe('mongo data types', () => {
     const collection = db.collection('test_data');
     await setupTable(db);
     await insert(collection);
+    await insertUndefined(db, 'test_data');
 
     const schema = await adapter.getConnectionSchema();
     const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0];
@@ -440,6 +495,10 @@ bucket_definitions:
 async function getReplicationTx(replicationStream: mongo.ChangeStream, count: number) {
   let transformed: SqliteRow[] = [];
   for await (const doc of replicationStream) {
+    // Specifically filter out map_input / map_output collections
+    if (!(doc as any)?.ns?.coll?.startsWith('test_data')) {
+      continue;
+    }
     transformed.push(constructAfterRecord((doc as any).fullDocument));
     if (transformed.length == count) {
       break;

package/test/src/setup.ts

@@ -1,7 +1,10 @@
 import { container } from '@powersync/lib-services-framework';
+import { test_utils } from '@powersync/service-core-tests';
 import { beforeAll } from 'vitest';
 
-beforeAll(() => {
+beforeAll(async () => {
   // Executes for every test file
   container.registerDefaults();
+
+  await test_utils.initMetrics();
 });

package/test/src/slow_tests.test.ts

@@ -1,30 +1,21 @@
-import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
-import { BucketStorageFactory } from '@powersync/service-core';
+import { storage } from '@powersync/service-core';
 import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
 import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js';
 import { env } from './env.js';
-
-type StorageFactory = () => Promise<BucketStorageFactory>;
-
-const BASIC_SYNC_RULES = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT _id as id, description FROM "test_data"
-`;
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () {
   if (env.CI || env.SLOW_TESTS) {
-    defineSlowTests(MONGO_STORAGE_FACTORY);
+    defineSlowTests(INITIALIZED_MONGO_STORAGE_FACTORY);
   } else {
     // Need something in this file.
     test('no-op', () => {});
   }
 });
 
-function defineSlowTests(factory: StorageFactory) {
+function defineSlowTests(factory: storage.TestStorageFactory) {
   test('replicating snapshot with lots of data', async () => {
     await using context = await ChangeStreamTestContext.open(factory);
     // Test with low minSnapshotHistoryWindowInSeconds, to trigger:
@@ -96,8 +87,10 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]', undefined, { limit: 50_000, chunkLimitBytes: 60_000_000 });
 
-    const preDocuments = data.filter((d) => JSON.parse(d.data! as string).description.startsWith('pre')).length;
-    const updatedDocuments = data.filter((d) => JSON.parse(d.data! as string).description.startsWith('updated')).length;
+    const preDocuments = data.filter((d: any) => JSON.parse(d.data! as string).description.startsWith('pre')).length;
+    const updatedDocuments = data.filter((d: any) =>
+      JSON.parse(d.data! as string).description.startsWith('updated')
+    ).length;
 
     // If the test works properly, preDocuments should be around 2000-3000.
     // The total should be around 9000-9900.

package/test/src/util.ts

@@ -1,18 +1,8 @@
 import * as types from '@module/types/types.js';
-import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId } from '@powersync/service-core';
 
-import { env } from './env.js';
-import { logger } from '@powersync/lib-services-framework';
-import { connectMongo } from '@core-tests/util.js';
+import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as mongo from 'mongodb';
-
-// The metrics need to be initialized before they can be used
-await Metrics.initialise({
-  disable_telemetry_sharing: true,
-  powersync_instance_id: 'test',
-  internal_metrics_endpoint: 'unused.for.tests.com'
-});
-Metrics.getInstance().resetCounters();
+import { env } from './env.js';
 
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
 
@@ -21,20 +11,10 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
   uri: TEST_URI
 });
 
-export type StorageFactory = () => Promise<BucketStorageFactory>;
-
-export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => {
-  const db = await connectMongo();
-
-  // None of the PG tests insert data into this collection, so it was never created
-  if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-    await db.db.createCollection('bucket_parameters');
-  }
-
-  await db.clear();
-
-  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
-};
+export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageFactoryGenerator({
+  url: env.MONGO_TEST_URL,
+  isCI: env.CI
+});
 
 export async function clearTestDb(db: mongo.Db) {
   await db.dropDatabase();

package/test/tsconfig.json

@@ -23,6 +23,9 @@
     },
     {
       "path": "../../../packages/service-core/"
+    },
+    {
+      "path": "../../../packages/service-core-tests/"
     }
   ]
 }

package/tsconfig.json

@@ -23,6 +23,12 @@
     },
     {
       "path": "../../libs/lib-services"
+    },
+    {
+      "path": "../../libs/lib-mongodb"
+    },
+    {
+      "path": "../module-mongodb-storage"
     }
   ]
 }