@powersync/service-module-mongodb 0.20.1 → 0.21.1

package/test/src/raw_change_stream.test.ts

@@ -5,7 +5,6 @@ import { getCursorBatchBytes } from '@module/replication/replication-index.js';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { bson } from '@powersync/service-core';
 import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
-import { testTimeout } from './test-timeouts.js';
 import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
 
 // DocumentDB only supports cluster-level change streams — collection- and
@@ -102,68 +101,6 @@ describe('internal mongodb utils', () => {
     }
   );
 
-  test(
-    'keeps getMore maxTimeMS when client-side maxAwaitTimeMS is enabled',
-    { timeout: testTimeout(30_000, { cloudOverride: 150_000 }) },
-    async () => {
-      const { db, client } = await connectMongoData({ monitorCommands: true });
-      await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
-      await clearTestDb(db);
-      const collection = db.collection('test_data');
-      // Keep the local Mongo path fast, but give Azure DocumentDB enough time
-      // for slow cloud change-stream delivery when maxTimeMS is sent to getMore.
-      const maxAwaitTimeMS = testTimeout(50, { cloudOverride: 10_000 });
-
-      const started: any[] = [];
-      client.on('commandStarted', (event) => {
-        if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
-          started.push(event);
-        }
-      });
-
-      const stream = rawChangeStream(
-        DATABASE_TYPE == DatabaseType.DOCUMENTDB ? client.db('admin') : db,
-        [
-          {
-            $changeStream: {
-              fullDocument: 'updateLookup',
-              ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB ? { allChangesForCluster: true } : {})
-            }
-          },
-          ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB
-            ? [
-                {
-                  $match: {
-                    'ns.db': db.databaseName,
-                    'ns.coll': collection.collectionName
-                  }
-                }
-              ]
-            : [])
-        ],
-        {
-          batchSize: 10,
-          maxAwaitTimeMS,
-          clientSideMaxAwaitTimeMS: true,
-          maxTimeMS: 1_000
-        }
-      );
-
-      await stream.next();
-      await collection.insertOne({ test: 1 });
-      const nextBatch = await readUntilNonEmptyBatch(stream);
-      await stream.return?.();
-
-      expect(nextBatch.events).toHaveLength(1);
-
-      const aggregate = started.find((event) => event.commandName == 'aggregate');
-      const getMore = started.find((event) => event.commandName == 'getMore');
-
-      expect(aggregate?.command.maxTimeMS).toEqual(1_000);
-      expect(getMore?.command.maxTimeMS).toEqual(maxAwaitTimeMS);
-    }
-  );
-
   // This test uses configureFailPoint to inject a getMore timeout. Azure
   // DocumentDB does not support configureFailPoint.
   test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(

package/test/src/schema.test.ts

@@ -0,0 +1,180 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+
+import { inferCollectionSchema } from '@module/api/infer-collection-schema.js';
+import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { testTimeout } from './test-timeouts.js';
+import { connectMongoData, requireFailCommand, TEST_CONNECTION_OPTIONS } from './util.js';
+
+const isDocumentDb = DATABASE_TYPE == DatabaseType.DOCUMENTDB;
+
+describe('collection schema inference', { timeout: testTimeout(20_000) }, () => {
+  let client: mongo.MongoClient;
+  let db: mongo.Db;
+
+  beforeEach(async () => {
+    ({ client } = await connectMongoData({ monitorCommands: true }));
+    db = client.db(`${TEST_CONNECTION_OPTIONS.database}_schema`);
+    await db.dropDatabase();
+  });
+
+  afterEach(async () => {
+    await db.dropDatabase();
+    await client.close();
+  });
+
+  test('merges types across documents and only infers top-level fields', async () => {
+    const collection = db.collection('mixed');
+    await collection.insertMany([
+      {
+        _id: 1 as any,
+        value: null,
+        whole: new mongo.Double(42),
+        huge: new mongo.Double(1e100),
+        fractional: new mongo.Double(-1.25),
+        nan: NaN,
+        positiveInfinity: Infinity,
+        negativeInfinity: -Infinity,
+        long: mongo.Long.fromString('9007199254740993'),
+        array: [{ nested: 'value' }],
+        object: { nested: true }
+      },
+      { _id: 2 as any, value: 'text', whole: new mongo.Int32(-42), array: [] },
+      { _id: 3 as any, value: 42, whole: new mongo.Double(-0) },
+      { _id: 4 as any, value: 1.25 },
+      { _id: 5 as any, value: new mongo.Double(42) }
+    ]);
+
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toMatchObject([
+      { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
+      { name: 'array', sqlite_type: 2, internal_type: 'Array' },
+      { name: 'fractional', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'huge', sqlite_type: 4, internal_type: 'Integer' },
+      { name: 'long', sqlite_type: 4, internal_type: 'Long' },
+      { name: 'nan', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'negativeInfinity', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'object', sqlite_type: 2, internal_type: 'Object' },
+      { name: 'positiveInfinity', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'value', sqlite_type: 14, internal_type: 'Double | Integer | Null | String' },
+      { name: 'whole', sqlite_type: 4, internal_type: 'Integer' }
+    ]);
+  });
+
+  test('distinguishes UUIDs from binary values by subtype and length', async () => {
+    const collection = db.collection('binary');
+    await collection.insertMany([
+      {
+        _id: 1 as any,
+        uuid: new mongo.UUID('00000000-0000-0000-0000-000000000000'),
+        binary: new mongo.Binary(Buffer.alloc(16)),
+        mixed: new mongo.UUID()
+      },
+      {
+        _id: 2 as any,
+        uuid: new mongo.UUID('ffffffff-ffff-ffff-ffff-ffffffffffff'),
+        binary: new mongo.Binary(Buffer.alloc(16, 255)),
+        mixed: new mongo.Binary(Buffer.alloc(16), mongo.Binary.SUBTYPE_UUID_OLD)
+      },
+      {
+        _id: 3 as any,
+        uuid: new mongo.UUID(),
+        binary: new mongo.Binary(Buffer.alloc(16), mongo.Binary.SUBTYPE_MD5)
+      },
+      { _id: 4 as any, binary: new mongo.Binary(Buffer.alloc(15, 255), mongo.Binary.SUBTYPE_USER_DEFINED) },
+      { _id: 5 as any, binary: new mongo.Binary(Buffer.alloc(17)) },
+      { _id: 6 as any, binary: new mongo.Binary(Buffer.alloc(0)) }
+    ]);
+
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toMatchObject([
+      { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
+      { name: 'binary', sqlite_type: 1, internal_type: 'Binary' },
+      { name: 'mixed', sqlite_type: 3, internal_type: 'Binary | UUID' },
+      { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' }
+    ]);
+  });
+
+  test('handles empty collections and documents with only an id', async () => {
+    const collection = await db.createCollection('empty');
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toEqual([]);
+
+    await collection.insertOne({});
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toEqual([
+      { name: '_id', sqlite_type: 2, type: 'ObjectId', internal_type: 'ObjectId', pg_type: 'ObjectId' }
+    ]);
+  });
+
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
+    'preserves field names with a non-simple collation',
+    async () => {
+      const collection = await db.createCollection('collation', { collation: { locale: 'en', strength: 1 } });
+      await collection.insertOne({ Name: 'text', name: 1, 'with.dot': true, $field: [] });
+
+      const columns = await inferCollectionSchema(collection, isDocumentDb);
+      expect(columns).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ name: 'Name', sqlite_type: 2 }),
+          expect.objectContaining({ name: 'name', sqlite_type: 4 }),
+          expect.objectContaining({ name: 'with.dot', sqlite_type: 4 }),
+          expect.objectContaining({ name: '$field', internal_type: 'Array' })
+        ])
+      );
+      expect(columns).toHaveLength(5);
+    }
+  );
+
+  test('returns only small schema metadata for multi-megabyte documents', async () => {
+    const collection = db.collection('large');
+    const largeString = 'x'.repeat(2 * 1024 * 1024);
+    for (let i = 0; i < 4; i++) {
+      await collection.insertOne({ text: largeString, binary: Buffer.alloc(2 * 1024 * 1024), array: [largeString] });
+    }
+
+    const responseSizes: number[] = [];
+    const executionLimits: number[] = [];
+    client.on('commandStarted', (event: mongo.CommandStartedEvent) => {
+      if (event.commandName == 'aggregate') {
+        executionLimits.push(event.command.maxTimeMS);
+      }
+    });
+    client.on('commandSucceeded', (event: mongo.CommandSucceededEvent) => {
+      if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
+        responseSizes.push(mongo.BSON.calculateObjectSize(event.reply as mongo.Document));
+      }
+    });
+
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toMatchObject([
+      { name: '_id', sqlite_type: 2, internal_type: 'ObjectId' },
+      { name: 'array', sqlite_type: 2, internal_type: 'Array' },
+      { name: 'binary', sqlite_type: 1, internal_type: 'Binary' },
+      { name: 'text', sqlite_type: 2, internal_type: 'String' }
+    ]);
+    expect(responseSizes.length).toBeGreaterThan(0);
+    expect(Math.max(...responseSizes)).toBeLessThan(4096);
+    expect(executionLimits).toEqual([30_000]);
+  });
+
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('fails schema inference on a query timeout', async (ctx) => {
+    await db.collection('timeout').insertOne({ value: 'text' });
+    await using adapter = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS,
+      database: db.databaseName
+    });
+    await using failCommand = await requireFailCommand(client, ctx);
+    await failCommand.configure({
+      mode: { times: 1 },
+      data: {
+        failCommands: ['aggregate'],
+        errorCode: 50 // MaxTimeMSExpired
+      }
+    });
+
+    // Exercise the real server/driver error path without waiting for the full timeout.
+    await expect(adapter.getConnectionSchema()).rejects.toMatchObject({
+      name: 'MongoServerError',
+      code: 50,
+      codeName: 'MaxTimeMSExpired'
+    });
+  });
+});

package/test/src/util.ts

@@ -3,13 +3,9 @@ import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 
 import * as types from '@module/types/types.js';
-import {
-  BSON_DESERIALIZE_DATA_OPTIONS,
-  SUPPORTED_STORAGE_VERSIONS,
-  TestStorageConfig,
-  TestStorageFactory
-} from '@powersync/service-core';
-import { describe, TestContext, TestOptions } from 'vitest';
+import { BSON_DESERIALIZE_DATA_OPTIONS, TestStorageFactory } from '@powersync/service-core';
+import { describeStorageCombinations } from '@powersync/service-core-tests';
+import { TestContext, TestOptions } from 'vitest';
 import { env } from './env.js';
 
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
@@ -29,34 +25,20 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.
   url: env.PG_STORAGE_TEST_URL
 });
 
-export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;
-
 export interface StorageVersionTestContext {
   factory: TestStorageFactory;
   storageVersion: number;
 }
 
 export function describeWithStorage(options: TestOptions, fn: (context: StorageVersionTestContext) => void) {
-  const describeFactory = (storageName: string, config: TestStorageConfig) => {
-    describe(`${storageName} storage`, options, function () {
-      for (const storageVersion of TEST_STORAGE_VERSIONS) {
-        describe(`storage v${storageVersion}`, function () {
-          fn({
-            factory: config.factory,
-            storageVersion
-          });
-        });
-      }
-    });
-  };
-
-  if (env.TEST_MONGO_STORAGE) {
-    describeFactory('mongodb', INITIALIZED_MONGO_STORAGE_FACTORY);
-  }
-
-  if (env.TEST_POSTGRES_STORAGE) {
-    describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY);
-  }
+  describeStorageCombinations(
+    {
+      mongodb: env.TEST_MONGO_STORAGE ? INITIALIZED_MONGO_STORAGE_FACTORY : undefined,
+      postgres: env.TEST_POSTGRES_STORAGE ? INITIALIZED_POSTGRES_STORAGE_FACTORY : undefined
+    },
+    options,
+    fn
+  );
 }
 
 export async function clearTestDb(db: mongo.Db) {