@powersync/service-module-mongodb 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/src/replication/checkpoints/create-checkpoint-implementation.ts

@@ -0,0 +1,14 @@
+import { CheckpointImplementation, CheckpointImplementationContext } from './CheckpointImplementation.js';
+import { SentinelCheckpointImplementation } from './SentinelCheckpointImplementation.js';
+import { TimestampCheckpointImplementation } from './TimestampCheckpointImplementation.js';
+
+/**
+ * Select the checkpoint implementation for a source: sentinel-based for DocumentDB
+ * DB, clusterTime-based for standard MongoDB.
+ */
+export function createCheckpointImplementation(
+  isDocumentDb: boolean,
+  context: CheckpointImplementationContext
+): CheckpointImplementation {
+  return isDocumentDb ? new SentinelCheckpointImplementation(context) : new TimestampCheckpointImplementation(context);
+}

package/src/replication/replication-utils.ts

@@ -1,3 +1,4 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as bson from 'bson';
 import { PostImagesOption } from '../types/types.js';
@@ -7,16 +8,36 @@ export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
 
 const REQUIRED_CHECKPOINT_PERMISSIONS = ['find', 'insert', 'update', 'remove', 'changeStream', 'createCollection'];
 
+/**
+ * Whether a `hello` response indicates Azure DocumentDB (formerly Azure Cosmos
+ * DB for MongoDB vCore), which reports `documentdb_versions` in the `internal`
+ * section.
+ */
+function isDocumentDbHello(hello: mongo.Document): boolean {
+  return hello.internal?.documentdb_versions != null;
+}
+
+/**
+ * Detect whether the connected server is DocumentDB. DocumentDB lacks usable
+ * clusterTime/operationTime and uses the sentinel checkpoint implementation.
+ */
+export async function detectDocumentDb(db: mongo.Db): Promise<boolean> {
+  const hello = await db.command({ hello: 1 });
+  return isDocumentDbHello(hello);
+}
+
 export async function checkSourceConfiguration(connectionManager: MongoManager): Promise<void> {
   const db = connectionManager.db;
 
   const hello = await db.command({ hello: 1 });
-  if (hello.msg == 'isdbgrid') {
+  const isDocumentDb = isDocumentDbHello(hello);
+
+  if (hello.msg == 'isdbgrid' && !isDocumentDb) {
     throw new ServiceError(
       ErrorCode.PSYNC_S1341,
       'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
     );
-  } else if (hello.setName == null) {
+  } else if (hello.setName == null && !isDocumentDb) {
     throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
   }
 

package/src/types/types.ts

@@ -1,5 +1,6 @@
 import type { MongoConnectionParams } from '@powersync/lib-service-mongodb/types';
 import * as lib_mongo from '@powersync/lib-service-mongodb/types';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as service_types from '@powersync/service-types';
 import { LookupFunction } from 'node:net';
 import * as t from 'ts-codec';
@@ -40,6 +41,10 @@ export enum PostImagesOption {
   READ_ONLY = 'read_only'
 }
 
+const DEFAULT_PING_INTERVAL_SECONDS = 60;
+const MIN_PING_INTERVAL_SECONDS = 5;
+const MAX_PING_INTERVAL_SECONDS = 60;
+
 export interface NormalizedMongoConnectionConfig {
   id: string;
   tag: string;
@@ -54,13 +59,19 @@ export interface NormalizedMongoConnectionConfig {
 
   postImages: PostImagesOption;
 
+  heartbeat_interval_seconds: number;
+
   connectionParams: MongoConnectionParams;
 }
 
 export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(
   t.object({
     // Replication specific settings
-    post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
+    post_images: service_types.enumLiteral('off', 'auto_configure', 'read_only').optional(),
+    /**
+     * Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
+     */
+    heartbeat_interval_seconds: service_types.orNull(t.number).optional()
   })
 );
 
@@ -87,6 +98,20 @@ export function normalizeConnectionConfig(options: MongoConnectionConfigDecoded)
     ...base,
     id: options.id ?? 'default',
     tag: options.tag ?? 'default',
-    postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF
+    postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF,
+    heartbeat_interval_seconds: normalizeHeartbeatInterval(options.heartbeat_interval_seconds)
   };
 }
+
+function normalizeHeartbeatInterval(value: number | null | undefined): number {
+  if (value == null) {
+    return DEFAULT_PING_INTERVAL_SECONDS;
+  }
+  if (!Number.isFinite(value) || value < MIN_PING_INTERVAL_SECONDS || value > MAX_PING_INTERVAL_SECONDS) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1109,
+      `MongoDB connection: heartbeat_interval_seconds must be between ${MIN_PING_INTERVAL_SECONDS} and ${MAX_PING_INTERVAL_SECONDS} seconds; null or omitted defaults to ${DEFAULT_PING_INTERVAL_SECONDS}`
+    );
+  }
+  return value;
+}

package/test/DOCUMENTDB_TESTING.md

@@ -0,0 +1,115 @@
+# Running Tests Against DocumentDB
+
+These instructions cover running the `module-mongodb` test suite against an Azure DocumentDB (formerly Cosmos DB for MongoDB vCore) cluster.
+
+## Prerequisites
+
+- An Azure DocumentDB (Cosmos DB for MongoDB vCore) cluster with change stream support
+- Local PostgreSQL for PowerSync's internal storage (not the source database)
+- The connection URI for the DocumentDB cluster
+
+> **The open-source `documentdb-local` Docker image cannot be used for these tests.** The
+> open-source DocumentDB engine does not implement change streams (`$changeStream is not
+supported yet in native pipeline`), does not report the `documentdb_versions` `hello` field
+> the suite uses for detection, and presents as `msg: isdbgrid`. An Azure-managed DocumentDB
+> (vCore) cluster is required.
+
+## Environment Variables
+
+DocumentDB is detected automatically from the server: the test suite runs
+`detectDocumentDb()` once at startup (see `DatabaseType.ts`) and gates the
+DocumentDB-specific tests on the result. There is no separate enable flag — pointing
+`MONGO_TEST_DATA_URL` at a DocumentDB cluster is what activates the DocumentDB tests.
+
+| Variable                  | Required | Description                                                                                                                                                              |
+| ------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `MONGO_TEST_DATA_URL`     | Yes      | DocumentDB connection URI. Must include a database name in the path (see below). Pointing this at a DocumentDB cluster enables the DocumentDB tests.                     |
+| `PG_STORAGE_TEST_URL`     | No       | PostgreSQL connection for PowerSync storage. Defaults to `postgres://postgres:postgres@localhost:5432/powersync_storage_test`.                                           |
+| `TEST_MONGO_STORAGE`      | No       | Set to `false` to skip MongoDB storage tests. Recommended when testing against DocumentDB to avoid using it as a storage backend.                                        |
+| `TEST_TIMEOUT_MULTIPLIER` | No       | Scale factor `testTimeout()` applies to a default timeout on DocumentDB when no explicit cloud override is given. Defaults to **6**. Increase for a slow/remote cluster. |
+
+### Connection URI format
+
+The `MONGO_TEST_DATA_URL` must include a database name in the path. DocumentDB URIs typically don't have one, so you need to add it before the query string:
+
+```
+# Original URI (no database):
+mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/
+
+# With database added:
+mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/powersync_test
+```
+
+If your password contains special characters (`=`, `@`, `+`, `/`), they must be URL-encoded in the URI (e.g., `=` becomes `%3D`). DocumentDB auto-generated passwords often contain `=` (base64).
+
+## Commands
+
+All commands run from the module directory: `modules/module-mongodb/`
+
+```bash
+# Run all DocumentDB tests (integration + unit helpers):
+MONGO_TEST_DATA_URL="mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/powersync_test" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb --reporter=verbose
+
+# Run only integration tests:
+MONGO_TEST_DATA_URL="<uri>" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb_mode --reporter=verbose
+
+# Run only unit helper tests (no DocumentDB cluster needed):
+npx vitest run documentdb_helpers --reporter=verbose
+
+# Run a specific test by name:
+MONGO_TEST_DATA_URL="<uri>" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb_mode -t "resume after restart" --reporter=verbose
+```
+
+If you have the URI in an environment variable (e.g., `$DOCUMENTDB_URI`), you can construct the test URL inline:
+
+```bash
+DOCUMENTDB_TEST_URL=$(echo "$DOCUMENTDB_URI" | sed 's|\?|powersync_test?|')
+MONGO_TEST_DATA_URL="$DOCUMENTDB_TEST_URL" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb --reporter=verbose
+```
+
+## GitHub Actions
+
+The `.github/workflows/documentdb-integration.yml` workflow runs these tests manually via `workflow_dispatch` only. Add a repository or organization secret named `AZURE_DOCUMENTDB_TEST_DATA_URL`, then dispatch the workflow. The workflow maps that secret to the test suite's `MONGO_TEST_DATA_URL` environment variable.
+
+The URI must include a database name in the path. The tests clear/drop this database as part of setup, so use a dedicated test database and cluster.
+
+The workflow starts a local MongoDB storage service, then runs the complete `modules/module-mongodb` `test` script against that storage backend. `MONGO_TEST_DATA_URL` is the remote DocumentDB source; `MONGO_TEST_URL` points at the local MongoDB storage test instance.
+
+```bash
+TEST_MONGO_STORAGE=true
+TEST_POSTGRES_STORAGE=false
+```
+
+## Test Files
+
+| File                         | Requires DocumentDB       | Description                                                                                                                                                                    |
+| ---------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `documentdb_mode.test.ts`    | Yes                       | Integration tests: replication, sentinel checkpoints, write checkpoints, keepalive, resume. Skipped automatically unless `MONGO_TEST_DATA_URL` points at a DocumentDB cluster. |
+| `documentdb_helpers.test.ts` | No (1 test needs MongoDB) | Unit tests: `getEventTimestamp`, sentinel parsing/matching, detection logic. Runs against any MongoDB or standalone.                                                           |
+
+## What the Integration Tests Cover
+
+Each integration test runs against 3 storage versions (v1, v2, v3) = 15 integration tests. Plus 15 unit tests in helpers = 30 total.
+
+| Test                    | What it validates                                                                               |
+| ----------------------- | ----------------------------------------------------------------------------------------------- |
+| basic replication       | Insert, update, delete through change stream with wallTime timestamps                           |
+| sentinel checkpoint     | Checkpoint created with `mode: 'sentinel'`, resolved by matching document content in the stream |
+| keepalive               | Stream idles past the keepalive interval without crashing on DocumentDB resume tokens           |
+| write checkpoint        | Full `createReplicationHead` → sentinel → polling flow for client write consistency             |
+| data events not dropped | Verifies `.lte()` dedup guard is skipped — events in the same wall-clock second are not lost    |
+| resume after restart    | Stop streaming, create new context, resume from stored token                                    |
+
+There is also a **characterization test**, `does not report collection drop and rename events`,
+that documents the current limitation: it writes DDL plus a post-DDL marker, waits for the marker
+(proving the stream caught up), then asserts no `drop` / `rename` events were delivered. It passes
+today and **fails if a future DocumentDB engine starts delivering DDL events** — a signal to add
+real drop/rename replication support.

package/test/src/DatabaseType.ts

@@ -0,0 +1,25 @@
+import { detectDocumentDb } from '@module/replication/replication-utils.js';
+import { logger } from '@powersync/lib-services-framework';
+import { connectMongoData } from './util.js';
+
+export enum DatabaseType {
+  DOCUMENTDB = 'DOCUMENTDB',
+  MONGODB = 'MONGODB'
+}
+
+let _databaseType: DatabaseType = DatabaseType.MONGODB;
+
+// Detected once at import time. Close the client afterwards so this detection
+// does not leak a connection: this module is imported by many test files, and
+// the client created here is otherwise never closed, accumulating open handles
+// in Vitest.
+const { client, db } = await connectMongoData();
+try {
+  _databaseType = (await detectDocumentDb(db)) ? DatabaseType.DOCUMENTDB : DatabaseType.MONGODB;
+} catch (ex) {
+  logger.warn(`Could not determine MongoDB database type`, ex);
+} finally {
+  await client.close().catch(() => {});
+}
+
+export const DATABASE_TYPE = _databaseType;