@powersync/service-module-mongodb 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/src/replication/checkpoints/TimestampCheckpointImplementation.ts

@@ -0,0 +1,144 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { ReplicationHeadCallback, storage } from '@powersync/service-core';
+import { MongoLSN } from '../../common/MongoLSN.js';
+import { createCheckpoint, SENTINEL_CHECKPOINT_ID, STANDALONE_CHECKPOINT_ID } from '../MongoRelation.js';
+import { CHECKPOINTS_COLLECTION, timestampToDate } from '../replication-utils.js';
+import {
+  CheckpointEventApi,
+  CheckpointImplementation,
+  CheckpointImplementationContext,
+  getCheckpointId,
+  getEventTimestamp,
+  StreamResumePosition
+} from './CheckpointImplementation.js';
+
+/**
+ * Standard MongoDB checkpoint implementation. The ordered LSN coordinate is the oplog
+ * clusterTime — unique per operation, monotonic, and parseable from resume
+ * tokens. Barriers and event LSNs are plain comparable LSN strings.
+ */
+export class TimestampCheckpointImplementation implements CheckpointImplementation {
+  readonly zeroLsn = MongoLSN.ZERO.comparable;
+
+  constructor(private context: CheckpointImplementationContext) {}
+
+  parseResumePosition(lsn: string): StreamResumePosition {
+    const parsed = MongoLSN.fromSerialized(lsn);
+    return { resumeAfter: parsed.resumeToken ?? null, startAfter: parsed.timestamp };
+  }
+
+  seedPosition(_lsn: string | null): void {
+    // The coordinate comes from each event's clusterTime; no state to seed.
+  }
+
+  logResume(lsn: string): void {
+    const parsed = MongoLSN.fromSerialized(lsn);
+    // It is normal for this to be a minute or two old when there is a low volume
+    // of ChangeStream events.
+    const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(parsed.timestamp).getTime()) / 1000);
+    this.context.logger.info(
+      `Resume streaming at ${parsed.timestamp.inspect()} / ${parsed} | Token age: ${tokenAgeSeconds}s`
+    );
+  }
+
+  async createStandaloneCheckpoint(): Promise<string> {
+    return createCheckpoint(this.context.db, STANDALONE_CHECKPOINT_ID);
+  }
+
+  async createBatchCheckpoint(): Promise<string> {
+    return createCheckpoint(this.context.db, this.context.checkpointStreamId);
+  }
+
+  async createFirstBarrier(): Promise<string | null> {
+    // The barrier marker is a comparable LSN; resume the snapshot stream from it.
+    return this.createBatchCheckpoint();
+  }
+
+  async keepalive(batch: storage.BucketStorageBatch, resumeToken: mongo.ResumeToken): Promise<void> {
+    // Parse the timestamp from the resume token. The ordered LSN prefix
+    // advances together with the token, so persisting is always safe.
+    const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(resumeToken);
+    await batch.keepalive(lsn);
+    // Log the token update. This helps as a general "replication is still active" message in the logs.
+    // This token would typically be around 10s behind.
+    this.context.logger.info(
+      `Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`
+    );
+  }
+
+  lsnFromResumeToken(resumeToken: mongo.ResumeToken): string {
+    // The timestamp is embedded in the resume token.
+    return MongoLSN.fromResumeToken(resumeToken).comparable;
+  }
+
+  async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
+    const session = this.context.client.startSession();
+    try {
+      await this.context.db.command({ hello: 1 }, { session });
+      const head = session.clusterTime?.clusterTime;
+      if (head == null) {
+        throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
+      }
+
+      const { response, shouldAdvance } = await callback(new MongoLSN({ timestamp: head }).comparable);
+
+      if (shouldAdvance) {
+        // Trigger a change on the changestream, so that the write checkpoint
+        // is processed without waiting for other writes.
+        await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
+          {
+            _id: STANDALONE_CHECKPOINT_ID as any
+          },
+          {
+            $inc: { i: 1 }
+          },
+          {
+            upsert: true,
+            returnDocument: 'after',
+            session
+          }
+        );
+        const time = session.operationTime!;
+        if (time == null) {
+          throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
+        } else if (time.lt(head)) {
+          throw new ServiceAssertionError(`operationTime must be > clusterTime`);
+        }
+      }
+
+      return response;
+    } finally {
+      await session.endSession();
+    }
+  }
+
+  readonly event: CheckpointEventApi = {
+    observe: (doc) => {
+      const checkpointId = getCheckpointId(doc);
+      if (checkpointId == null || checkpointId == SENTINEL_CHECKPOINT_ID) {
+        return 'foreign';
+      }
+      // The STANDALONE_CHECKPOINT_ID is only used for the TimestampCheckpointImplementation
+      if (checkpointId == STANDALONE_CHECKPOINT_ID) {
+        return 'standalone';
+      }
+      return this.context.checkpointStreamId.equals(checkpointId) ? 'own-barrier' : 'foreign';
+    },
+
+    lsn: (doc) => {
+      return new MongoLSN({
+        timestamp: getEventTimestamp(doc),
+        resume_token: doc._id
+      }).comparable;
+    },
+
+    resolvesBarrier: (marker, doc) => {
+      // Barrier markers are comparable LSNs in this implementation.
+      return this.event.lsn(doc) >= marker;
+    }
+  };
+
+  // It's safe to clear the entire _powersync_checkpoints collection in this mode.
+  readonly checkpointClearFilter: mongo.Filter<mongo.Document> = {};
+}

package/src/replication/checkpoints/create-checkpoint-implementation.ts

@@ -0,0 +1,14 @@
+import { CheckpointImplementation, CheckpointImplementationContext } from './CheckpointImplementation.js';
+import { SentinelCheckpointImplementation } from './SentinelCheckpointImplementation.js';
+import { TimestampCheckpointImplementation } from './TimestampCheckpointImplementation.js';
+
+/**
+ * Select the checkpoint implementation for a source: sentinel-based for DocumentDB
+ * DB, clusterTime-based for standard MongoDB.
+ */
+export function createCheckpointImplementation(
+  isDocumentDb: boolean,
+  context: CheckpointImplementationContext
+): CheckpointImplementation {
+  return isDocumentDb ? new SentinelCheckpointImplementation(context) : new TimestampCheckpointImplementation(context);
+}

package/src/replication/replication-utils.ts

@@ -1,3 +1,4 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as bson from 'bson';
 import { PostImagesOption } from '../types/types.js';
@@ -7,16 +8,36 @@ export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
 
 const REQUIRED_CHECKPOINT_PERMISSIONS = ['find', 'insert', 'update', 'remove', 'changeStream', 'createCollection'];
 
+/**
+ * Whether a `hello` response indicates Azure DocumentDB (formerly Azure Cosmos
+ * DB for MongoDB vCore), which reports `documentdb_versions` in the `internal`
+ * section.
+ */
+function isDocumentDbHello(hello: mongo.Document): boolean {
+  return hello.internal?.documentdb_versions != null;
+}
+
+/**
+ * Detect whether the connected server is DocumentDB. DocumentDB lacks usable
+ * clusterTime/operationTime and uses the sentinel checkpoint implementation.
+ */
+export async function detectDocumentDb(db: mongo.Db): Promise<boolean> {
+  const hello = await db.command({ hello: 1 });
+  return isDocumentDbHello(hello);
+}
+
 export async function checkSourceConfiguration(connectionManager: MongoManager): Promise<void> {
   const db = connectionManager.db;
 
   const hello = await db.command({ hello: 1 });
-  if (hello.msg == 'isdbgrid') {
+  const isDocumentDb = isDocumentDbHello(hello);
+
+  if (hello.msg == 'isdbgrid' && !isDocumentDb) {
     throw new ServiceError(
       ErrorCode.PSYNC_S1341,
       'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
     );
-  } else if (hello.setName == null) {
+  } else if (hello.setName == null && !isDocumentDb) {
     throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
   }
 

package/test/DOCUMENTDB_TESTING.md

@@ -0,0 +1,115 @@
+# Running Tests Against DocumentDB
+
+These instructions cover running the `module-mongodb` test suite against an Azure DocumentDB (formerly Cosmos DB for MongoDB vCore) cluster.
+
+## Prerequisites
+
+- An Azure DocumentDB (Cosmos DB for MongoDB vCore) cluster with change stream support
+- Local PostgreSQL for PowerSync's internal storage (not the source database)
+- The connection URI for the DocumentDB cluster
+
+> **The open-source `documentdb-local` Docker image cannot be used for these tests.** The
+> open-source DocumentDB engine does not implement change streams (`$changeStream is not
+supported yet in native pipeline`), does not report the `documentdb_versions` `hello` field
+> the suite uses for detection, and presents as `msg: isdbgrid`. An Azure-managed DocumentDB
+> (vCore) cluster is required.
+
+## Environment Variables
+
+DocumentDB is detected automatically from the server: the test suite runs
+`detectDocumentDb()` once at startup (see `DatabaseType.ts`) and gates the
+DocumentDB-specific tests on the result. There is no separate enable flag — pointing
+`MONGO_TEST_DATA_URL` at a DocumentDB cluster is what activates the DocumentDB tests.
+
+| Variable                  | Required | Description                                                                                                                                                              |
+| ------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `MONGO_TEST_DATA_URL`     | Yes      | DocumentDB connection URI. Must include a database name in the path (see below). Pointing this at a DocumentDB cluster enables the DocumentDB tests.                     |
+| `PG_STORAGE_TEST_URL`     | No       | PostgreSQL connection for PowerSync storage. Defaults to `postgres://postgres:postgres@localhost:5432/powersync_storage_test`.                                           |
+| `TEST_MONGO_STORAGE`      | No       | Set to `false` to skip MongoDB storage tests. Recommended when testing against DocumentDB to avoid using it as a storage backend.                                        |
+| `TEST_TIMEOUT_MULTIPLIER` | No       | Scale factor `testTimeout()` applies to a default timeout on DocumentDB when no explicit cloud override is given. Defaults to **6**. Increase for a slow/remote cluster. |
+
+### Connection URI format
+
+The `MONGO_TEST_DATA_URL` must include a database name in the path. DocumentDB URIs typically don't have one, so you need to add it before the query string:
+
+```
+# Original URI (no database):
+mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/
+
+# With database added:
+mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/powersync_test
+```
+
+If your password contains special characters (`=`, `@`, `+`, `/`), they must be URL-encoded in the URI (e.g., `=` becomes `%3D`). DocumentDB auto-generated passwords often contain `=` (base64).
+
+## Commands
+
+All commands run from the module directory: `modules/module-mongodb/`
+
+```bash
+# Run all DocumentDB tests (integration + unit helpers):
+MONGO_TEST_DATA_URL="mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/powersync_test" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb --reporter=verbose
+
+# Run only integration tests:
+MONGO_TEST_DATA_URL="<uri>" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb_mode --reporter=verbose
+
+# Run only unit helper tests (no DocumentDB cluster needed):
+npx vitest run documentdb_helpers --reporter=verbose
+
+# Run a specific test by name:
+MONGO_TEST_DATA_URL="<uri>" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb_mode -t "resume after restart" --reporter=verbose
+```
+
+If you have the URI in an environment variable (e.g., `$DOCUMENTDB_URI`), you can construct the test URL inline:
+
+```bash
+DOCUMENTDB_TEST_URL=$(echo "$DOCUMENTDB_URI" | sed 's|\?|powersync_test?|')
+MONGO_TEST_DATA_URL="$DOCUMENTDB_TEST_URL" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb --reporter=verbose
+```
+
+## GitHub Actions
+
+The `.github/workflows/documentdb-integration.yml` workflow runs these tests manually via `workflow_dispatch` only. Add a repository or organization secret named `AZURE_DOCUMENTDB_TEST_DATA_URL`, then dispatch the workflow. The workflow maps that secret to the test suite's `MONGO_TEST_DATA_URL` environment variable.
+
+The URI must include a database name in the path. The tests clear/drop this database as part of setup, so use a dedicated test database and cluster.
+
+The workflow starts a local MongoDB storage service, then runs the complete `modules/module-mongodb` `test` script against that storage backend. `MONGO_TEST_DATA_URL` is the remote DocumentDB source; `MONGO_TEST_URL` points at the local MongoDB storage test instance.
+
+```bash
+TEST_MONGO_STORAGE=true
+TEST_POSTGRES_STORAGE=false
+```
+
+## Test Files
+
+| File                         | Requires DocumentDB       | Description                                                                                                                                                                    |
+| ---------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `documentdb_mode.test.ts`    | Yes                       | Integration tests: replication, sentinel checkpoints, write checkpoints, keepalive, resume. Skipped automatically unless `MONGO_TEST_DATA_URL` points at a DocumentDB cluster. |
+| `documentdb_helpers.test.ts` | No (1 test needs MongoDB) | Unit tests: `getEventTimestamp`, sentinel parsing/matching, detection logic. Runs against any MongoDB or standalone.                                                           |
+
+## What the Integration Tests Cover
+
+Each integration test runs against 3 storage versions (v1, v2, v3) = 15 integration tests. Plus 15 unit tests in helpers = 30 total.
+
+| Test                    | What it validates                                                                               |
+| ----------------------- | ----------------------------------------------------------------------------------------------- |
+| basic replication       | Insert, update, delete through change stream with wallTime timestamps                           |
+| sentinel checkpoint     | Checkpoint created with `mode: 'sentinel'`, resolved by matching document content in the stream |
+| keepalive               | Stream idles past the keepalive interval without crashing on DocumentDB resume tokens           |
+| write checkpoint        | Full `createReplicationHead` → sentinel → polling flow for client write consistency             |
+| data events not dropped | Verifies `.lte()` dedup guard is skipped — events in the same wall-clock second are not lost    |
+| resume after restart    | Stop streaming, create new context, resume from stored token                                    |
+
+There is also a **characterization test**, `does not report collection drop and rename events`,
+that documents the current limitation: it writes DDL plus a post-DDL marker, waits for the marker
+(proving the stream caught up), then asserts no `drop` / `rename` events were delivered. It passes
+today and **fails if a future DocumentDB engine starts delivering DDL events** — a signal to add
+real drop/rename replication support.

package/test/src/DatabaseType.ts

@@ -0,0 +1,25 @@
+import { detectDocumentDb } from '@module/replication/replication-utils.js';
+import { logger } from '@powersync/lib-services-framework';
+import { connectMongoData } from './util.js';
+
+export enum DatabaseType {
+  DOCUMENTDB = 'DOCUMENTDB',
+  MONGODB = 'MONGODB'
+}
+
+let _databaseType: DatabaseType = DatabaseType.MONGODB;
+
+// Detected once at import time. Close the client afterwards so this detection
+// does not leak a connection: this module is imported by many test files, and
+// the client created here is otherwise never closed, accumulating open handles
+// in Vitest.
+const { client, db } = await connectMongoData();
+try {
+  _databaseType = (await detectDocumentDb(db)) ? DatabaseType.DOCUMENTDB : DatabaseType.MONGODB;
+} catch (ex) {
+  logger.warn(`Could not determine MongoDB database type`, ex);
+} finally {
+  await client.close().catch(() => {});
+}
+
+export const DATABASE_TYPE = _databaseType;