@powersync/service-module-mongodb 0.1.8 → 0.3.0

package/src/api/MongoRouteAPIAdapter.ts

@@ -1,13 +1,14 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { mongo } from '@powersync/lib-service-mongodb';
 import { api, ParseSyncRulesOptions, SourceTable } from '@powersync/service-core';
-import * as mongo from 'mongodb';
-
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as service_types from '@powersync/service-types';
+
 import { MongoManager } from '../replication/MongoManager.js';
 import { constructAfterRecord, createCheckpoint } from '../replication/MongoRelation.js';
+import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js';
 import * as types from '../types/types.js';
 import { escapeRegExp } from '../utils.js';
-import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js';
 
 export class MongoRouteAPIAdapter implements api.RouteAPI {
   protected client: mongo.MongoClient;
@@ -45,7 +46,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
   async getConnectionStatus(): Promise<service_types.ConnectionStatusV2> {
     const base = {
       id: this.config.id,
-      uri: types.baseUri(this.config)
+      uri: lib_mongo.baseUri(this.config)
     };
 
     try {
@@ -225,7 +226,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
         try {
           collections = await this.client.db(db.name).listCollections().toArray();
         } catch (e) {
-          if (e instanceof mongo.MongoServerError && e.codeName == 'Unauthorized') {
+          if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
             // Ignore databases we're not authorized to query
             return null;
           }
@@ -267,7 +268,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
               });
             }
           } catch (e) {
-            if (e instanceof mongo.MongoServerError && e.codeName == 'Unauthorized') {
+            if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
               // Ignore collections we're not authorized to query
               continue;
             }

package/src/index.ts

@@ -1 +1,4 @@
+export * from './api/MongoRouteAPIAdapter.js';
 export * from './module/MongoModule.js';
+export * from './replication/replication-index.js';
+export * from './types/types.js';

package/src/module/MongoModule.ts

@@ -1,25 +1,22 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { api, ConfigurationFileSyncRulesProvider, replication, system, TearDownOptions } from '@powersync/service-core';
 import { MongoRouteAPIAdapter } from '../api/MongoRouteAPIAdapter.js';
+import { ChangeStreamReplicator } from '../replication/ChangeStreamReplicator.js';
 import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactory.js';
 import { MongoErrorRateLimiter } from '../replication/MongoErrorRateLimiter.js';
-import { ChangeStreamReplicator } from '../replication/ChangeStreamReplicator.js';
-import * as types from '../types/types.js';
 import { MongoManager } from '../replication/MongoManager.js';
 import { checkSourceConfiguration } from '../replication/replication-utils.js';
+import * as types from '../types/types.js';
 
 export class MongoModule extends replication.ReplicationModule<types.MongoConnectionConfig> {
   constructor() {
     super({
       name: 'MongoDB',
-      type: types.MONGO_CONNECTION_TYPE,
+      type: lib_mongo.MONGO_CONNECTION_TYPE,
       configSchema: types.MongoConnectionConfig
     });
   }
 
-  async initialize(context: system.ServiceContextContainer): Promise<void> {
-    await super.initialize(context);
-  }
-
   protected createRouteAPIAdapter(): api.RouteAPI {
     return new MongoRouteAPIAdapter(this.resolveConfig(this.decodedConfig!));
   }

package/src/replication/ChangeStream.ts

@@ -1,7 +1,9 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { container, logger } from '@powersync/lib-services-framework';
 import { Metrics, SaveOperationTag, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core';
 import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
-import * as mongo from 'mongodb';
+import { PostImagesOption } from '../types/types.js';
+import { escapeRegExp } from '../utils.js';
 import { MongoManager } from './MongoManager.js';
 import {
   constructAfterRecord,
@@ -10,9 +12,7 @@ import {
   getMongoRelation,
   mongoLsnToTimestamp
 } from './MongoRelation.js';
-import { escapeRegExp } from '../utils.js';
 import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
-import { PostImagesOption } from '../types/types.js';
 
 export const ZERO_LSN = '0000000000000000';
 

package/src/replication/ChangeStreamReplicationJob.ts

@@ -1,10 +1,9 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { container } from '@powersync/lib-services-framework';
-import { ChangeStreamInvalidatedError, ChangeStream } from './ChangeStream.js';
-
 import { replication } from '@powersync/service-core';
-import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
 
-import * as mongo from 'mongodb';
+import { ChangeStream, ChangeStreamInvalidatedError } from './ChangeStream.js';
+import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
 
 export interface ChangeStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions {
   connectionFactory: ConnectionManagerFactory;

package/src/replication/MongoManager.ts

@@ -1,4 +1,5 @@
-import * as mongo from 'mongodb';
+import { mongo } from '@powersync/lib-service-mongodb';
+
 import { NormalizedMongoConnectionConfig } from '../types/types.js';
 
 export class MongoManager {

package/src/replication/MongoRelation.ts

@@ -1,7 +1,8 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
-import { SqliteRow, SqliteValue, toSyncRulesRow } from '@powersync/service-sync-rules';
-import * as mongo from 'mongodb';
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
+import { SqliteRow, SqliteValue } from '@powersync/service-sync-rules';
+
 import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
 
 export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor {
@@ -38,9 +39,12 @@ export function constructAfterRecord(document: mongo.Document): SqliteRow {
 
 export function toMongoSyncRulesValue(data: any): SqliteValue {
   const autoBigNum = true;
-  if (data == null) {
-    // null or undefined
-    return data;
+  if (data === null) {
+    return null;
+  } else if (typeof data == 'undefined') {
+    // We consider `undefined` in top-level fields as missing replicated value,
+    // so use null instead.
+    return null;
   } else if (typeof data == 'string') {
     return data;
   } else if (typeof data == 'number') {
@@ -95,8 +99,13 @@ function filterJsonData(data: any, depth = 0): any {
     // This is primarily to prevent infinite recursion
     throw new Error(`json nested object depth exceeds the limit of ${DEPTH_LIMIT}`);
   }
-  if (data == null) {
-    return data; // null or undefined
+  if (data === null) {
+    return data;
+  } else if (typeof data == 'undefined') {
+    // For nested data, keep as undefined.
+    // In arrays, this is converted to null.
+    // In objects, the key is excluded.
+    return undefined;
   } else if (typeof data == 'string') {
     return data;
   } else if (typeof data == 'number') {

package/src/types/types.ts

@@ -1,9 +1,7 @@
-import { normalizeMongoConfig } from '@powersync/service-core';
+import * as lib_mongo from '@powersync/lib-service-mongodb/types';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
 
-export const MONGO_CONNECTION_TYPE = 'mongodb' as const;
-
 export enum PostImagesOption {
   /**
    * Use fullDocument: updateLookup on the changeStream.
@@ -53,18 +51,9 @@ export interface NormalizedMongoConnectionConfig {
   postImages: PostImagesOption;
 }
 
-export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(
+export const MongoConnectionConfig = lib_mongo.BaseMongoConfig.and(service_types.configFile.DataSourceConfig).and(
   t.object({
-    type: t.literal(MONGO_CONNECTION_TYPE),
-    /** Unique identifier for the connection - optional when a single connection is present. */
-    id: t.string.optional(),
-    /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */
-    tag: t.string.optional(),
-    uri: t.string,
-    username: t.string.optional(),
-    password: t.string.optional(),
-    database: t.string.optional(),
-
+    // Replication specific settings
     post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
   })
 );
@@ -72,20 +61,21 @@ export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.a
 /**
  * Config input specified when starting services
  */
-export type MongoConnectionConfig = t.Decoded<typeof MongoConnectionConfig>;
+export type MongoConnectionConfig = t.Encoded<typeof MongoConnectionConfig>;
+export type MongoConnectionConfigDecoded = t.Decoded<typeof MongoConnectionConfig>;
 
 /**
  * Resolved version of {@link MongoConnectionConfig}
  */
-export type ResolvedConnectionConfig = MongoConnectionConfig & NormalizedMongoConnectionConfig;
+export type ResolvedConnectionConfig = MongoConnectionConfigDecoded & NormalizedMongoConnectionConfig;
 
 /**
  * Validate and normalize connection options.
  *
  * Returns destructured options.
  */
-export function normalizeConnectionConfig(options: MongoConnectionConfig): NormalizedMongoConnectionConfig {
-  const base = normalizeMongoConfig(options);
+export function normalizeConnectionConfig(options: MongoConnectionConfigDecoded): NormalizedMongoConnectionConfig {
+  const base = lib_mongo.normalizeMongoConfig(options);
 
   return {
     ...base,
@@ -94,12 +84,3 @@ export function normalizeConnectionConfig(options: MongoConnectionConfig): Norma
     postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF
   };
 }
-
-/**
- * Construct a mongodb URI, without username, password or ssl options.
- *
- * Only contains hostname, port, database.
- */
-export function baseUri(options: NormalizedMongoConnectionConfig) {
-  return options.uri;
-}

package/test/src/change_stream.test.ts

@@ -1,14 +1,14 @@
-import { putOp, removeOp } from '@core-tests/stream_utils.js';
-import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
-import { BucketStorageFactory } from '@powersync/service-core';
 import * as crypto from 'crypto';
-import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test, vi } from 'vitest';
-import { ChangeStreamTestContext } from './change_stream_utils.js';
-import { PostImagesOption } from '@module/types/types.js';
 
-type StorageFactory = () => Promise<BucketStorageFactory>;
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { test_utils } from '@powersync/service-core-tests';
+
+import { PostImagesOption } from '@module/types/types.js';
+import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:
@@ -18,10 +18,10 @@ bucket_definitions:
 `;
 
 describe('change stream - mongodb', { timeout: 20_000 }, function () {
-  defineChangeStreamTests(MONGO_STORAGE_FACTORY);
+  defineChangeStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 });
 
-function defineChangeStreamTests(factory: StorageFactory) {
+function defineChangeStreamTests(factory: storage.TestStorageFactory) {
   test('replicating basic values', async () => {
     await using context = await ChangeStreamTestContext.open(factory);
     const { db } = context;
@@ -52,10 +52,10 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id.toHexString())
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id.toHexString())
     ]);
   });
 
@@ -86,8 +86,8 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n })
     ]);
   });
 
@@ -125,11 +125,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // fullDocument is not available at the point this is replicated, resulting in it treated as a remove
-      removeOp('test_data', test_id!.toHexString()),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.removeOp('test_data', test_id!.toHexString()),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -171,11 +171,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // The postImage helps us get this data
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -216,11 +216,11 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }),
       // The postImage helps us get this data
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
-      removeOp('test_data', test_id!.toHexString())
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2', num: 1152921504606846976n }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }),
+      test_utils.removeOp('test_data', test_id!.toHexString())
     ]);
   });
 
@@ -244,7 +244,7 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
 
-    expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
+    expect(data).toMatchObject([test_utils.putOp('test_DATA', { id: test_id, description: 'test1' })]);
   });
 
   test('replicating large values', async () => {
@@ -270,10 +270,10 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
     expect(data.slice(0, 1)).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription })
     ]);
     expect(data.slice(1)).toMatchObject([
-      putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription })
+      test_utils.putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription })
     ]);
   });
 
@@ -302,8 +302,8 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id, description: 'test1' }),
-      removeOp('test_data', test_id)
+      test_utils.putOp('test_data', { id: test_id, description: 'test1' }),
+      test_utils.removeOp('test_data', test_id)
     ]);
   });
 
@@ -330,9 +330,9 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([
-      putOp('test_data1', { id: test_id, description: 'test1' }),
-      removeOp('test_data1', test_id),
-      putOp('test_data2', { id: test_id, description: 'test1' })
+      test_utils.putOp('test_data1', { id: test_id, description: 'test1' }),
+      test_utils.removeOp('test_data1', test_id),
+      test_utils.putOp('test_data2', { id: test_id, description: 'test1' })
     ]);
   });
 
@@ -349,7 +349,7 @@ bucket_definitions:
     context.startStreaming();
 
     const data = await context.getBucketData('global[]');
-    expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
+    expect(data).toMatchObject([test_utils.putOp('test_data', { id: test_id, description: 'test1' })]);
   });
 
   test('large record', async () => {
@@ -446,8 +446,8 @@ bucket_definitions:
 
     const data = await context.getBucketData('global[]');
     expect(data).toMatchObject([
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
-      putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+      test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
     ]);
   });
 

package/test/src/change_stream_utils.ts

@@ -1,13 +1,14 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core';
+import { test_utils } from '@powersync/service-core-tests';
 
-import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
-import { fromAsync } from '@core-tests/stream_utils.js';
-import { MongoManager } from '@module/replication/MongoManager.js';
 import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
-import * as mongo from 'mongodb';
+import { MongoManager } from '@module/replication/MongoManager.js';
 import { createCheckpoint } from '@module/replication/MongoRelation.js';
 import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
 
+import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
+
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
   private abortController = new AbortController();
@@ -102,7 +103,7 @@ export class ChangeStreamTestContext {
   async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
     const map = new Map<string, string>(Object.entries(buckets));
-    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+    return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
   async getBucketData(
@@ -117,7 +118,7 @@ export class ChangeStreamTestContext {
       limit: options?.limit,
       chunkLimitBytes: options?.chunkLimitBytes
     });
-    const batches = await fromAsync(batch);
+    const batches = await test_utils.fromAsync(batch);
     return batches[0]?.batch.data ?? [];
   }
 

package/test/src/env.ts

@@ -1,6 +1,7 @@
 import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
+  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'),
   CI: utils.type.boolean.default('false'),
   SLOW_TESTS: utils.type.boolean.default('false')

package/test/src/mongo_test.test.ts

@@ -1,11 +1,12 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+
 import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
 import { ChangeStream } from '@module/replication/ChangeStream.js';
 import { constructAfterRecord } from '@module/replication/MongoRelation.js';
-import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
-import * as mongo from 'mongodb';
-import { describe, expect, test } from 'vitest';
-import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js';
 import { PostImagesOption } from '@module/types/types.js';
+import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js';
 
 describe('mongo data types', () => {
   async function setupTable(db: mongo.Db) {
@@ -45,12 +46,52 @@ describe('mongo data types', () => {
           mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'),
           'mydb',
           { foo: 'bar' }
-        ),
-        undefined: undefined
+        )
       }
     ]);
   }
 
+  async function insertUndefined(db: mongo.Db, collection: string, array?: boolean) {
+    // MongoDB has deprecated the `undefined` value, making it really
+    // difficult to insert one into the database.
+    // mapReduce is also deprecated, but it's one way to still generate
+    // the value.
+    const mapInput = db.collection('map_input');
+    await mapInput.insertOne({ test: 'test' });
+    const fin = array ? `return { result: [undefined] }` : `return { result: undefined }`;
+    await db.command({
+      mapReduce: 'map_input',
+      map: new mongo.Code(`function () {
+          // We only need to emit once for a single result:
+          emit(5, {});
+        }`),
+      reduce: new mongo.Code(`function (key, values) {
+          // Return an object whose property is explicitly set to undefined
+          return undefined;
+        }`),
+      finalize: new mongo.Code(`function (key, reducedVal) {
+          ${fin};
+        }`),
+      out: { merge: 'map_output' }
+    });
+
+    await db
+      .collection('map_output')
+      .aggregate([
+        { $set: { undefined: '$value.result' } },
+        { $project: { undefined: 1 } },
+        {
+          $merge: {
+            into: collection
+          }
+        }
+      ])
+      .toArray();
+
+    await mapInput.drop();
+    await db.collection('map_output').drop();
+  }
+
   async function insertNested(collection: mongo.Collection) {
     await collection.insertMany([
       {
@@ -118,9 +159,11 @@ describe('mongo data types', () => {
       js: '{"code":"testcode","scope":null}',
       js2: '{"code":"testcode","scope":{"foo":"bar"}}',
       pointer: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}',
-      pointer2: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}',
-      undefined: null
+      pointer2: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}'
     });
+
+    // This must specifically be null, and not undefined.
+    expect(transformed[4].undefined).toBeNull();
   }
 
   function checkResultsNested(transformed: Record<string, any>[]) {
@@ -158,20 +201,27 @@ describe('mongo data types', () => {
       js: '[{"code":"testcode","scope":null}]',
       pointer: '[{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}]',
       minKey: '[null]',
-      maxKey: '[null]',
+      maxKey: '[null]'
+    });
+
+    expect(transformed[4]).toMatchObject({
       undefined: '[null]'
     });
   }
 
   test('test direct queries', async () => {
     const { db, client } = await connectMongoData();
+
     const collection = db.collection('test_data');
     try {
       await setupTable(db);
-
       await insert(collection);
+      await insertUndefined(db, 'test_data');
 
-      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data').find().toArray())];
+      const rawResults = await db.collection('test_data').find().toArray();
+      // It is tricky to save "undefined" with mongo, so we check that it succeeded.
+      expect(rawResults[4].undefined).toBeUndefined();
+      const transformed = [...ChangeStream.getQueryData(rawResults)];
 
       checkResults(transformed);
     } finally {
@@ -186,8 +236,11 @@ describe('mongo data types', () => {
       await setupTable(db);
 
       await insertNested(collection);
+      await insertUndefined(db, 'test_data_arrays', true);
 
-      const transformed = [...ChangeStream.getQueryData(await db.collection('test_data_arrays').find().toArray())];
+      const rawResults = await db.collection('test_data_arrays').find().toArray();
+      expect(rawResults[4].undefined).toEqual([undefined]);
+      const transformed = [...ChangeStream.getQueryData(rawResults)];
 
       checkResultsNested(transformed);
     } finally {
@@ -212,8 +265,9 @@ describe('mongo data types', () => {
       await stream.tryNext();
 
       await insert(collection);
+      await insertUndefined(db, 'test_data');
 
-      const transformed = await getReplicationTx(stream, 4);
+      const transformed = await getReplicationTx(stream, 5);
 
       checkResults(transformed);
     } finally {
@@ -236,8 +290,9 @@ describe('mongo data types', () => {
       await stream.tryNext();
 
       await insertNested(collection);
+      await insertUndefined(db, 'test_data_arrays', true);
 
-      const transformed = await getReplicationTx(stream, 4);
+      const transformed = await getReplicationTx(stream, 5);
 
       checkResultsNested(transformed);
     } finally {
@@ -256,6 +311,7 @@ describe('mongo data types', () => {
     const collection = db.collection('test_data');
     await setupTable(db);
     await insert(collection);
+    await insertUndefined(db, 'test_data');
 
     const schema = await adapter.getConnectionSchema();
     const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0];
@@ -440,6 +496,10 @@ bucket_definitions:
 async function getReplicationTx(replicationStream: mongo.ChangeStream, count: number) {
   let transformed: SqliteRow[] = [];
   for await (const doc of replicationStream) {
+    // Specifically filter out map_input / map_output collections
+    if (!(doc as any)?.ns?.coll?.startsWith('test_data')) {
+      continue;
+    }
     transformed.push(constructAfterRecord((doc as any).fullDocument));
     if (transformed.length == count) {
       break;

package/test/src/setup.ts

@@ -1,7 +1,10 @@
 import { container } from '@powersync/lib-services-framework';
+import { test_utils } from '@powersync/service-core-tests';
 import { beforeAll } from 'vitest';
 
-beforeAll(() => {
+beforeAll(async () => {
   // Executes for every test file
   container.registerDefaults();
+
+  await test_utils.initMetrics();
 });