@powersync/service-module-mongodb 0.0.0-dev-20260803144812 → 0.0.0-dev-20260909133214

package/src/replication/RawChangeStream.ts

@@ -6,14 +6,8 @@ import {
   ReplicationAssertionError
 } from '@powersync/lib-services-framework';
 import { PerformanceTracer } from '@powersync/service-core';
-import { performance } from 'node:perf_hooks';
-import { setTimeout as delay } from 'node:timers/promises';
 import { ChangeStreamInvalidatedError } from './ChangeStream.js';
 
-// Keep the DocumentDB idle-poll workaround from adding the full maxAwaitTimeMS
-// as local latency when an update arrives just after an empty batch.
-const CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS = 1_000;
-
 export interface RawChangeStreamOptions {
   signal?: AbortSignal;
 
@@ -21,21 +15,12 @@ export interface RawChangeStreamOptions {
    * How long to wait for new data per batch (max time for long-polling).
    * This is sent as maxTimeMS for the getMore command.
    *
-   * A value of 0 is allowed for probe-style streams that do not want an idle
-   * wait; in that case PowerSync also skips the local empty-batch delay.
+   * A value of 0 removes the explicit await limit and leaves the server's
+   * default awaitData behavior in effect; it does not make getMore return
+   * immediately. Snapshot probes use 0 for this purpose.
    */
   maxAwaitTimeMS: number;
 
-  /**
-   * Also enforce maxAwaitTimeMS on the client for empty getMore batches.
-   *
-   * Azure DocumentDB currently returns idle getMore calls before maxTimeMS. When
-   * this is enabled, empty batches are delayed locally (capped at 1s) to avoid
-   * tight polling. We still send maxTimeMS so this remains compatible with
-   * servers that handle maxAwaitTimeMS correctly.
-   */
-  clientSideMaxAwaitTimeMS?: boolean;
-
   /**
    * Timeout for the initial aggregate command.
    */
@@ -230,17 +215,12 @@ async function* rawChangeStreamInner(
       options.signal?.throwIfAborted();
 
       using commandSpan = options.tracer?.span('changestream', 'getmore');
-      const getMoreStartedAt = performance.now();
       const getMoreCommand: mongo.Document = {
         getMore: cursorId,
         collection: nsCollection,
         batchSize: batchSizer.next(),
         maxTimeMS: options.maxAwaitTimeMS
       };
-      // Azure DocumentDB currently returns empty getMore batches before
-      // maxTimeMS expires. Keep maxTimeMS for forward compatibility with the
-      // server-side behavior, and when client-side mode is enabled, enforce the
-      // capped idle wait locally for empty batches below.
       const getMoreResult: mongo.Document = await db.command(getMoreCommand, { session, raw: true }).catch((e) => {
         if (isMongoServerError(e) && e.codeName == 'CursorKilled') {
           // This may be due to the killCursors command issued when aborting.
@@ -268,13 +248,6 @@ async function* rawChangeStreamInner(
         // postBatchResumeToken is returned in MongoDB 4.0.7 and later, and we support 6.0+
         throw new ReplicationAssertionError(`postBatchResumeToken from aggregate response`);
       }
-      if (options.clientSideMaxAwaitTimeMS && nextBatch.length == 0) {
-        const remainingMaxAwaitTimeMS = Math.ceil(options.maxAwaitTimeMS - (performance.now() - getMoreStartedAt));
-        if (remainingMaxAwaitTimeMS > 0) {
-          const clientSideDelayMs = Math.min(remainingMaxAwaitTimeMS, CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS);
-          await delay(clientSideDelayMs, undefined, { signal: options.signal });
-        }
-      }
       yield {
         events: nextBatch,
         resumeToken: cursor.postBatchResumeToken,

package/src/replication/checkpoints/CheckpointImplementation.ts

@@ -128,7 +128,7 @@ export interface CheckpointImplementation {
    * The timestamp implementation parses the timestamp embedded in the token.
    * The sentinel implementation pairs the token with the current coordinate.
    */
-  lsnFromResumeToken(resumeToken: mongo.ResumeToken): string;
+  lsnFromResumeToken(resumeToken: mongo.ResumeToken): { lsn: string; timestamp: Date | null };
 
   /**
    * Source-side replication head for write checkpoints. The LSN passed to the

package/src/replication/checkpoints/SentinelCheckpointImplementation.ts

@@ -108,11 +108,11 @@ export class SentinelCheckpointImplementation implements CheckpointImplementatio
     );
   }
 
-  lsnFromResumeToken(resumeToken: mongo.ResumeToken): string {
+  lsnFromResumeToken(resumeToken: mongo.ResumeToken) {
     // Pair the bare token with the current coordinate. The coordinate is
     // frozen between checkpoint events, so a per-batch resume marker only
     // advances the token; that is all resumption needs (see the design doc).
-    return new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable;
+    return { lsn: new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable, timestamp: null };
   }
 
   async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {

package/src/replication/checkpoints/TimestampCheckpointImplementation.ts

@@ -67,9 +67,10 @@ export class TimestampCheckpointImplementation implements CheckpointImplementati
     );
   }
 
-  lsnFromResumeToken(resumeToken: mongo.ResumeToken): string {
+  lsnFromResumeToken(resumeToken: mongo.ResumeToken) {
     // The timestamp is embedded in the resume token.
-    return MongoLSN.fromResumeToken(resumeToken).comparable;
+    const lsn = MongoLSN.fromResumeToken(resumeToken);
+    return { lsn: lsn.comparable, timestamp: timestampToDate(lsn.timestamp) };
   }
 
   async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {

package/src/types/types.ts

@@ -1,5 +1,6 @@
 import type { MongoConnectionParams } from '@powersync/lib-service-mongodb/types';
 import * as lib_mongo from '@powersync/lib-service-mongodb/types';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as service_types from '@powersync/service-types';
 import { LookupFunction } from 'node:net';
 import * as t from 'ts-codec';
@@ -40,6 +41,10 @@ export enum PostImagesOption {
   READ_ONLY = 'read_only'
 }
 
+const DEFAULT_PING_INTERVAL_SECONDS = 60;
+const MIN_PING_INTERVAL_SECONDS = 5;
+const MAX_PING_INTERVAL_SECONDS = 60;
+
 export interface NormalizedMongoConnectionConfig {
   id: string;
   tag: string;
@@ -54,13 +59,19 @@ export interface NormalizedMongoConnectionConfig {
 
   postImages: PostImagesOption;
 
+  heartbeat_interval_seconds: number;
+
   connectionParams: MongoConnectionParams;
 }
 
 export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(
   t.object({
     // Replication specific settings
-    post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
+    post_images: service_types.enumLiteral('off', 'auto_configure', 'read_only').optional(),
+    /**
+     * Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
+     */
+    heartbeat_interval_seconds: service_types.orNull(t.number).optional()
   })
 );
 
@@ -87,6 +98,20 @@ export function normalizeConnectionConfig(options: MongoConnectionConfigDecoded)
     ...base,
     id: options.id ?? 'default',
     tag: options.tag ?? 'default',
-    postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF
+    postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF,
+    heartbeat_interval_seconds: normalizeHeartbeatInterval(options.heartbeat_interval_seconds)
   };
 }
+
+function normalizeHeartbeatInterval(value: number | null | undefined): number {
+  if (value == null) {
+    return DEFAULT_PING_INTERVAL_SECONDS;
+  }
+  if (!Number.isFinite(value) || value < MIN_PING_INTERVAL_SECONDS || value > MAX_PING_INTERVAL_SECONDS) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1109,
+      `MongoDB connection: heartbeat_interval_seconds must be between ${MIN_PING_INTERVAL_SECONDS} and ${MAX_PING_INTERVAL_SECONDS} seconds; null or omitted defaults to ${DEFAULT_PING_INTERVAL_SECONDS}`
+    );
+  }
+  return value;
+}

package/test/src/change_stream.test.ts

@@ -622,6 +622,79 @@ bucket_definitions:
     expect(data).toMatchObject([test_utils.putOp('test_data', { id: test_id, description: 'test1' })]);
   });
 
+  test.runIf(storageVersion >= 3)(
+    'does not resnapshot an unchanged event definition during incremental reprocessing',
+    async (testContext) => {
+      const eventDefinition = `
+event_definitions:
+  checkpoint_requests:
+    payloads:
+      - SELECT user_id, checkpoint, client_id, CASE WHEN checkpoint_requested_at IS NULL THEN true ELSE false END AS is_legacy FROM checkpoints
+`;
+      const syncConfig = (deploymentComment: string) => `
+config:
+  edition: 3
+  storage_version: 3
+  # ${deploymentComment}
+
+${eventDefinition}
+streams:
+  global:
+    auto_subscribe: true
+    queries:
+      - SELECT *, _id AS id FROM lists
+      - SELECT *, _id AS id FROM todos
+`;
+
+      let replicationStreamId: number;
+      {
+        await using context = await openContext();
+        const firstStorage = await context.updateSyncRules(syncConfig('initial deployment'));
+        if (
+          !('incrementalReprocessing' in firstStorage.storageConfig) ||
+          firstStorage.storageConfig.incrementalReprocessing !== true
+        ) {
+          testContext.skip('storage does not support incremental reprocessing');
+        }
+
+        const { db } = context;
+        await db.collection('lists').insertOne({ name: 'list' });
+        await db.collection('todos').insertOne({ description: 'todo' });
+        await db.collection('checkpoints').insertOne({
+          user_id: 'user-1',
+          checkpoint: 1n,
+          client_id: 'client-1',
+          checkpoint_requested_at: null
+        });
+
+        replicationStreamId = firstStorage.replicationStreamId;
+        await context.replicateSnapshot();
+        context.startStreaming();
+        await context.getCheckpoint();
+      }
+
+      const resnapshottedCollections: string[] = [];
+      await using context = await openContext({
+        doNotClear: true,
+        streamOptions: {
+          snapshotHooks: {
+            beforeSnapshotStarted: async (table) => {
+              resnapshottedCollections.push(table.name);
+            }
+          }
+        }
+      });
+
+      const processingStorage = await context.updateSyncRules(syncConfig('redeployed without behavioral changes'));
+      expect(processingStorage.replicationStreamId).toBe(replicationStreamId);
+      expect(await context.factory.getDeployingSyncConfig()).not.toBeNull();
+
+      await context.replicateSnapshot();
+
+      expect(resnapshottedCollections).toEqual([]);
+    }
+  );
+
   test('coalesces standalone checkpoints when backlog is buffered', async () => {
     await using context = await openContext();
     await context.updateSyncRules(BASIC_SYNC_RULES);

package/test/src/config.test.ts

@@ -0,0 +1,34 @@
+import { normalizeConnectionConfig } from '@module/types/types.js';
+import { describe, expect, test } from 'vitest';
+
+const BASE_CONFIG = {
+  type: 'mongodb' as const,
+  uri: 'mongodb://localhost:27017/powersync_test'
+};
+
+describe('MongoDB connection config', () => {
+  test('defaults heartbeat_interval_seconds to 60 seconds', () => {
+    expect(normalizeConnectionConfig(BASE_CONFIG).heartbeat_interval_seconds).toBe(60);
+  });
+
+  test('defaults a null heartbeat_interval_seconds to 60 seconds', () => {
+    expect(
+      normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: null }).heartbeat_interval_seconds
+    ).toBe(60);
+  });
+
+  test('allows the MongoDB maximum heartbeat interval', () => {
+    expect(
+      normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: 60 }).heartbeat_interval_seconds
+    ).toBe(60);
+  });
+
+  test.each([0, 4, 61, Number.NaN, Number.POSITIVE_INFINITY])(
+    'rejects invalid heartbeat_interval_seconds: %s',
+    (heartbeat_interval_seconds) => {
+      expect(() => normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds })).toThrow(
+        'heartbeat_interval_seconds'
+      );
+    }
+  );
+});

package/test/src/documentdb_mode.test.ts

@@ -759,8 +759,7 @@ bucket_definitions:
     expect(JSON.parse(lastOp.data as string)).toMatchObject({ description: 'after_keepalive' });
   });
 
-  // Skipped until Azure DocumentDB ships the server-side getMore maxAwaitTimeMS fix.
-  test.skip('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
+  test('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
     const maxAwaitTimeMS = 2_000;
 
     await using context = await openContext({

package/test/src/raw_change_stream.test.ts

@@ -5,7 +5,6 @@ import { getCursorBatchBytes } from '@module/replication/replication-index.js';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { bson } from '@powersync/service-core';
 import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
-import { testTimeout } from './test-timeouts.js';
 import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
 
 // DocumentDB only supports cluster-level change streams — collection- and
@@ -102,68 +101,6 @@ describe('internal mongodb utils', () => {
     }
   );
 
-  test(
-    'keeps getMore maxTimeMS when client-side maxAwaitTimeMS is enabled',
-    { timeout: testTimeout(30_000, { cloudOverride: 150_000 }) },
-    async () => {
-      const { db, client } = await connectMongoData({ monitorCommands: true });
-      await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
-      await clearTestDb(db);
-      const collection = db.collection('test_data');
-      // Keep the local Mongo path fast, but give Azure DocumentDB enough time
-      // for slow cloud change-stream delivery when maxTimeMS is sent to getMore.
-      const maxAwaitTimeMS = testTimeout(50, { cloudOverride: 10_000 });
-
-      const started: any[] = [];
-      client.on('commandStarted', (event) => {
-        if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
-          started.push(event);
-        }
-      });
-
-      const stream = rawChangeStream(
-        DATABASE_TYPE == DatabaseType.DOCUMENTDB ? client.db('admin') : db,
-        [
-          {
-            $changeStream: {
-              fullDocument: 'updateLookup',
-              ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB ? { allChangesForCluster: true } : {})
-            }
-          },
-          ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB
-            ? [
-                {
-                  $match: {
-                    'ns.db': db.databaseName,
-                    'ns.coll': collection.collectionName
-                  }
-                }
-              ]
-            : [])
-        ],
-        {
-          batchSize: 10,
-          maxAwaitTimeMS,
-          clientSideMaxAwaitTimeMS: true,
-          maxTimeMS: 1_000
-        }
-      );
-
-      await stream.next();
-      await collection.insertOne({ test: 1 });
-      const nextBatch = await readUntilNonEmptyBatch(stream);
-      await stream.return?.();
-
-      expect(nextBatch.events).toHaveLength(1);
-
-      const aggregate = started.find((event) => event.commandName == 'aggregate');
-      const getMore = started.find((event) => event.commandName == 'getMore');
-
-      expect(aggregate?.command.maxTimeMS).toEqual(1_000);
-      expect(getMore?.command.maxTimeMS).toEqual(maxAwaitTimeMS);
-    }
-  );
-
   // This test uses configureFailPoint to inject a getMore timeout. Azure
   // DocumentDB does not support configureFailPoint.
   test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(

package/test/src/util.ts

@@ -3,13 +3,9 @@ import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 
 import * as types from '@module/types/types.js';
-import {
-  BSON_DESERIALIZE_DATA_OPTIONS,
-  SUPPORTED_STORAGE_VERSIONS,
-  TestStorageConfig,
-  TestStorageFactory
-} from '@powersync/service-core';
-import { describe, TestContext, TestOptions } from 'vitest';
+import { BSON_DESERIALIZE_DATA_OPTIONS, TestStorageFactory } from '@powersync/service-core';
+import { describeStorageCombinations } from '@powersync/service-core-tests';
+import { TestContext, TestOptions } from 'vitest';
 import { env } from './env.js';
 
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
@@ -29,34 +25,20 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.
   url: env.PG_STORAGE_TEST_URL
 });
 
-export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;
-
 export interface StorageVersionTestContext {
   factory: TestStorageFactory;
   storageVersion: number;
 }
 
 export function describeWithStorage(options: TestOptions, fn: (context: StorageVersionTestContext) => void) {
-  const describeFactory = (storageName: string, config: TestStorageConfig) => {
-    describe(`${storageName} storage`, options, function () {
-      for (const storageVersion of TEST_STORAGE_VERSIONS) {
-        describe(`storage v${storageVersion}`, function () {
-          fn({
-            factory: config.factory,
-            storageVersion
-          });
-        });
-      }
-    });
-  };
-
-  if (env.TEST_MONGO_STORAGE) {
-    describeFactory('mongodb', INITIALIZED_MONGO_STORAGE_FACTORY);
-  }
-
-  if (env.TEST_POSTGRES_STORAGE) {
-    describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY);
-  }
+  describeStorageCombinations(
+    {
+      mongodb: env.TEST_MONGO_STORAGE ? INITIALIZED_MONGO_STORAGE_FACTORY : undefined,
+      postgres: env.TEST_POSTGRES_STORAGE ? INITIALIZED_POSTGRES_STORAGE_FACTORY : undefined
+    },
+    options,
+    fn
+  );
 }
 
 export async function clearTestDb(db: mongo.Db) {