@powersync/service-module-mongodb 0.21.0 → 0.21.1

package/src/replication/ChangeStream.ts

@@ -213,7 +213,7 @@ export class ChangeStream {
     this.isDocumentDb = await detectDocumentDb(this.defaultDb);
     if (this.isDocumentDb) {
       this.logger.warn(
-        'Azure DocumentDB support is experimental. APIs and behavior may change, and long-term stability is not yet guaranteed.'
+        'Azure DocumentDB support is in alpha. APIs and behavior may change, and long-term stability is not yet guaranteed.'
       );
     }
     this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
@@ -595,9 +595,6 @@ export class ChangeStream {
     return rawChangeStream(watchDb, pipeline, {
       batchSize: options.batchSize ?? this.snapshotChunkLength,
       maxAwaitTimeMS,
-      // maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
-      // In that case there is no client-side wait to emulate for DocumentDB.
-      clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
       maxTimeMS: this.changeStreamTimeout,
 
       signal: options.signal,

package/src/replication/MongoSnapshotter.ts

@@ -785,9 +785,6 @@ export class MongoSnapshotter {
     return rawChangeStream(watchDb, pipeline, {
       batchSize: options.batchSize ?? this.snapshotChunkLength,
       maxAwaitTimeMS,
-      // maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
-      // In that case there is no client-side wait to emulate for DocumentDB.
-      clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
       maxTimeMS: this.changeStreamTimeout,
       signal: options.signal,
       logger: this.logger,

package/src/replication/RawChangeStream.ts

@@ -6,14 +6,8 @@ import {
   ReplicationAssertionError
 } from '@powersync/lib-services-framework';
 import { PerformanceTracer } from '@powersync/service-core';
-import { performance } from 'node:perf_hooks';
-import { setTimeout as delay } from 'node:timers/promises';
 import { ChangeStreamInvalidatedError } from './ChangeStream.js';
 
-// Keep the DocumentDB idle-poll workaround from adding the full maxAwaitTimeMS
-// as local latency when an update arrives just after an empty batch.
-const CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS = 1_000;
-
 export interface RawChangeStreamOptions {
   signal?: AbortSignal;
 
@@ -21,21 +15,12 @@ export interface RawChangeStreamOptions {
    * How long to wait for new data per batch (max time for long-polling).
    * This is sent as maxTimeMS for the getMore command.
    *
-   * A value of 0 is allowed for probe-style streams that do not want an idle
-   * wait; in that case PowerSync also skips the local empty-batch delay.
+   * A value of 0 removes the explicit await limit and leaves the server's
+   * default awaitData behavior in effect; it does not make getMore return
+   * immediately. Snapshot probes use 0 for this purpose.
    */
   maxAwaitTimeMS: number;
 
-  /**
-   * Also enforce maxAwaitTimeMS on the client for empty getMore batches.
-   *
-   * Azure DocumentDB currently returns idle getMore calls before maxTimeMS. When
-   * this is enabled, empty batches are delayed locally (capped at 1s) to avoid
-   * tight polling. We still send maxTimeMS so this remains compatible with
-   * servers that handle maxAwaitTimeMS correctly.
-   */
-  clientSideMaxAwaitTimeMS?: boolean;
-
   /**
    * Timeout for the initial aggregate command.
    */
@@ -230,17 +215,12 @@ async function* rawChangeStreamInner(
       options.signal?.throwIfAborted();
 
       using commandSpan = options.tracer?.span('changestream', 'getmore');
-      const getMoreStartedAt = performance.now();
       const getMoreCommand: mongo.Document = {
         getMore: cursorId,
         collection: nsCollection,
         batchSize: batchSizer.next(),
         maxTimeMS: options.maxAwaitTimeMS
       };
-      // Azure DocumentDB currently returns empty getMore batches before
-      // maxTimeMS expires. Keep maxTimeMS for forward compatibility with the
-      // server-side behavior, and when client-side mode is enabled, enforce the
-      // capped idle wait locally for empty batches below.
       const getMoreResult: mongo.Document = await db.command(getMoreCommand, { session, raw: true }).catch((e) => {
         if (isMongoServerError(e) && e.codeName == 'CursorKilled') {
           // This may be due to the killCursors command issued when aborting.
@@ -268,13 +248,6 @@ async function* rawChangeStreamInner(
         // postBatchResumeToken is returned in MongoDB 4.0.7 and later, and we support 6.0+
         throw new ReplicationAssertionError(`postBatchResumeToken from aggregate response`);
       }
-      if (options.clientSideMaxAwaitTimeMS && nextBatch.length == 0) {
-        const remainingMaxAwaitTimeMS = Math.ceil(options.maxAwaitTimeMS - (performance.now() - getMoreStartedAt));
-        if (remainingMaxAwaitTimeMS > 0) {
-          const clientSideDelayMs = Math.min(remainingMaxAwaitTimeMS, CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS);
-          await delay(clientSideDelayMs, undefined, { signal: options.signal });
-        }
-      }
       yield {
         events: nextBatch,
         resumeToken: cursor.postBatchResumeToken,

package/test/src/documentdb_mode.test.ts

@@ -759,8 +759,7 @@ bucket_definitions:
     expect(JSON.parse(lastOp.data as string)).toMatchObject({ description: 'after_keepalive' });
   });
 
-  // Skipped until Azure DocumentDB ships the server-side getMore maxAwaitTimeMS fix.
-  test.skip('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
+  test('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
     const maxAwaitTimeMS = 2_000;
 
     await using context = await openContext({

package/test/src/raw_change_stream.test.ts

@@ -5,7 +5,6 @@ import { getCursorBatchBytes } from '@module/replication/replication-index.js';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { bson } from '@powersync/service-core';
 import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
-import { testTimeout } from './test-timeouts.js';
 import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
 
 // DocumentDB only supports cluster-level change streams — collection- and
@@ -102,68 +101,6 @@ describe('internal mongodb utils', () => {
     }
   );
 
-  test(
-    'keeps getMore maxTimeMS when client-side maxAwaitTimeMS is enabled',
-    { timeout: testTimeout(30_000, { cloudOverride: 150_000 }) },
-    async () => {
-      const { db, client } = await connectMongoData({ monitorCommands: true });
-      await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
-      await clearTestDb(db);
-      const collection = db.collection('test_data');
-      // Keep the local Mongo path fast, but give Azure DocumentDB enough time
-      // for slow cloud change-stream delivery when maxTimeMS is sent to getMore.
-      const maxAwaitTimeMS = testTimeout(50, { cloudOverride: 10_000 });
-
-      const started: any[] = [];
-      client.on('commandStarted', (event) => {
-        if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
-          started.push(event);
-        }
-      });
-
-      const stream = rawChangeStream(
-        DATABASE_TYPE == DatabaseType.DOCUMENTDB ? client.db('admin') : db,
-        [
-          {
-            $changeStream: {
-              fullDocument: 'updateLookup',
-              ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB ? { allChangesForCluster: true } : {})
-            }
-          },
-          ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB
-            ? [
-                {
-                  $match: {
-                    'ns.db': db.databaseName,
-                    'ns.coll': collection.collectionName
-                  }
-                }
-              ]
-            : [])
-        ],
-        {
-          batchSize: 10,
-          maxAwaitTimeMS,
-          clientSideMaxAwaitTimeMS: true,
-          maxTimeMS: 1_000
-        }
-      );
-
-      await stream.next();
-      await collection.insertOne({ test: 1 });
-      const nextBatch = await readUntilNonEmptyBatch(stream);
-      await stream.return?.();
-
-      expect(nextBatch.events).toHaveLength(1);
-
-      const aggregate = started.find((event) => event.commandName == 'aggregate');
-      const getMore = started.find((event) => event.commandName == 'getMore');
-
-      expect(aggregate?.command.maxTimeMS).toEqual(1_000);
-      expect(getMore?.command.maxTimeMS).toEqual(maxAwaitTimeMS);
-    }
-  );
-
   // This test uses configureFailPoint to inject a getMore timeout. Azure
   // DocumentDB does not support configureFailPoint.
   test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(

package/test/src/schema.test.ts

@@ -0,0 +1,180 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+
+import { inferCollectionSchema } from '@module/api/infer-collection-schema.js';
+import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { testTimeout } from './test-timeouts.js';
+import { connectMongoData, requireFailCommand, TEST_CONNECTION_OPTIONS } from './util.js';
+
+const isDocumentDb = DATABASE_TYPE == DatabaseType.DOCUMENTDB;
+
+describe('collection schema inference', { timeout: testTimeout(20_000) }, () => {
+  let client: mongo.MongoClient;
+  let db: mongo.Db;
+
+  beforeEach(async () => {
+    ({ client } = await connectMongoData({ monitorCommands: true }));
+    db = client.db(`${TEST_CONNECTION_OPTIONS.database}_schema`);
+    await db.dropDatabase();
+  });
+
+  afterEach(async () => {
+    await db.dropDatabase();
+    await client.close();
+  });
+
+  test('merges types across documents and only infers top-level fields', async () => {
+    const collection = db.collection('mixed');
+    await collection.insertMany([
+      {
+        _id: 1 as any,
+        value: null,
+        whole: new mongo.Double(42),
+        huge: new mongo.Double(1e100),
+        fractional: new mongo.Double(-1.25),
+        nan: NaN,
+        positiveInfinity: Infinity,
+        negativeInfinity: -Infinity,
+        long: mongo.Long.fromString('9007199254740993'),
+        array: [{ nested: 'value' }],
+        object: { nested: true }
+      },
+      { _id: 2 as any, value: 'text', whole: new mongo.Int32(-42), array: [] },
+      { _id: 3 as any, value: 42, whole: new mongo.Double(-0) },
+      { _id: 4 as any, value: 1.25 },
+      { _id: 5 as any, value: new mongo.Double(42) }
+    ]);
+
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toMatchObject([
+      { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
+      { name: 'array', sqlite_type: 2, internal_type: 'Array' },
+      { name: 'fractional', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'huge', sqlite_type: 4, internal_type: 'Integer' },
+      { name: 'long', sqlite_type: 4, internal_type: 'Long' },
+      { name: 'nan', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'negativeInfinity', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'object', sqlite_type: 2, internal_type: 'Object' },
+      { name: 'positiveInfinity', sqlite_type: 8, internal_type: 'Double' },
+      { name: 'value', sqlite_type: 14, internal_type: 'Double | Integer | Null | String' },
+      { name: 'whole', sqlite_type: 4, internal_type: 'Integer' }
+    ]);
+  });
+
+  test('distinguishes UUIDs from binary values by subtype and length', async () => {
+    const collection = db.collection('binary');
+    await collection.insertMany([
+      {
+        _id: 1 as any,
+        uuid: new mongo.UUID('00000000-0000-0000-0000-000000000000'),
+        binary: new mongo.Binary(Buffer.alloc(16)),
+        mixed: new mongo.UUID()
+      },
+      {
+        _id: 2 as any,
+        uuid: new mongo.UUID('ffffffff-ffff-ffff-ffff-ffffffffffff'),
+        binary: new mongo.Binary(Buffer.alloc(16, 255)),
+        mixed: new mongo.Binary(Buffer.alloc(16), mongo.Binary.SUBTYPE_UUID_OLD)
+      },
+      {
+        _id: 3 as any,
+        uuid: new mongo.UUID(),
+        binary: new mongo.Binary(Buffer.alloc(16), mongo.Binary.SUBTYPE_MD5)
+      },
+      { _id: 4 as any, binary: new mongo.Binary(Buffer.alloc(15, 255), mongo.Binary.SUBTYPE_USER_DEFINED) },
+      { _id: 5 as any, binary: new mongo.Binary(Buffer.alloc(17)) },
+      { _id: 6 as any, binary: new mongo.Binary(Buffer.alloc(0)) }
+    ]);
+
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toMatchObject([
+      { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
+      { name: 'binary', sqlite_type: 1, internal_type: 'Binary' },
+      { name: 'mixed', sqlite_type: 3, internal_type: 'Binary | UUID' },
+      { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' }
+    ]);
+  });
+
+  test('handles empty collections and documents with only an id', async () => {
+    const collection = await db.createCollection('empty');
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toEqual([]);
+
+    await collection.insertOne({});
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toEqual([
+      { name: '_id', sqlite_type: 2, type: 'ObjectId', internal_type: 'ObjectId', pg_type: 'ObjectId' }
+    ]);
+  });
+
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
+    'preserves field names with a non-simple collation',
+    async () => {
+      const collection = await db.createCollection('collation', { collation: { locale: 'en', strength: 1 } });
+      await collection.insertOne({ Name: 'text', name: 1, 'with.dot': true, $field: [] });
+
+      const columns = await inferCollectionSchema(collection, isDocumentDb);
+      expect(columns).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ name: 'Name', sqlite_type: 2 }),
+          expect.objectContaining({ name: 'name', sqlite_type: 4 }),
+          expect.objectContaining({ name: 'with.dot', sqlite_type: 4 }),
+          expect.objectContaining({ name: '$field', internal_type: 'Array' })
+        ])
+      );
+      expect(columns).toHaveLength(5);
+    }
+  );
+
+  test('returns only small schema metadata for multi-megabyte documents', async () => {
+    const collection = db.collection('large');
+    const largeString = 'x'.repeat(2 * 1024 * 1024);
+    for (let i = 0; i < 4; i++) {
+      await collection.insertOne({ text: largeString, binary: Buffer.alloc(2 * 1024 * 1024), array: [largeString] });
+    }
+
+    const responseSizes: number[] = [];
+    const executionLimits: number[] = [];
+    client.on('commandStarted', (event: mongo.CommandStartedEvent) => {
+      if (event.commandName == 'aggregate') {
+        executionLimits.push(event.command.maxTimeMS);
+      }
+    });
+    client.on('commandSucceeded', (event: mongo.CommandSucceededEvent) => {
+      if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
+        responseSizes.push(mongo.BSON.calculateObjectSize(event.reply as mongo.Document));
+      }
+    });
+
+    expect(await inferCollectionSchema(collection, isDocumentDb)).toMatchObject([
+      { name: '_id', sqlite_type: 2, internal_type: 'ObjectId' },
+      { name: 'array', sqlite_type: 2, internal_type: 'Array' },
+      { name: 'binary', sqlite_type: 1, internal_type: 'Binary' },
+      { name: 'text', sqlite_type: 2, internal_type: 'String' }
+    ]);
+    expect(responseSizes.length).toBeGreaterThan(0);
+    expect(Math.max(...responseSizes)).toBeLessThan(4096);
+    expect(executionLimits).toEqual([30_000]);
+  });
+
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('fails schema inference on a query timeout', async (ctx) => {
+    await db.collection('timeout').insertOne({ value: 'text' });
+    await using adapter = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS,
+      database: db.databaseName
+    });
+    await using failCommand = await requireFailCommand(client, ctx);
+    await failCommand.configure({
+      mode: { times: 1 },
+      data: {
+        failCommands: ['aggregate'],
+        errorCode: 50 // MaxTimeMSExpired
+      }
+    });
+
+    // Exercise the real server/driver error path without waiting for the full timeout.
+    await expect(adapter.getConnectionSchema()).rejects.toMatchObject({
+      name: 'MongoServerError',
+      code: 50,
+      codeName: 'MaxTimeMSExpired'
+    });
+  });
+});