@powersync/service-module-mongodb 0.18.2 → 0.20.0

package/src/replication/checkpoints/SentinelCheckpointImplementation.ts

@@ -0,0 +1,269 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationHeadCallback, storage } from '@powersync/service-core';
+import { JSONBig } from '@powersync/service-jsonbig';
+import { SentinelLSN } from '../../common/SentinelLSN.js';
+import { ChangeStreamInvalidatedError } from '../ChangeStream.js';
+import { createSentinelCheckpointLsn, SENTINEL_CHECKPOINT_ID } from '../MongoRelation.js';
+import { ProjectedChangeStreamDocument } from '../RawChangeStream.js';
+import {
+  CheckpointEventApi,
+  CheckpointEventKind,
+  CheckpointImplementation,
+  CheckpointImplementationContext,
+  getCheckpointId,
+  StreamResumePosition
+} from './CheckpointImplementation.js';
+
+/**
+ * Sentinel checkpoint implementation, used for sources without a usable clusterTime
+ * (DocumentDB). The ordered LSN coordinate is a single shared sentinel checkpoint
+ * counter ({@link SENTINEL_CHECKPOINT_ID}), observed through the change stream:
+ *
+ * - Batch checkpoints advance the global counter and stamp it with this stream's
+ *   id, so the stream recognises its own private barriers by content (stream_id
+ *   + counter) rather than by cross-document event ordering.
+ * - Standalone bumps (write checkpoint heads, snapshot markers) advance the same
+ *   counter with a null stream_id; every stream observes these as the global
+ *   coordinate.
+ * - Keepalives advance the counter without persisting; the bump's own event
+ *   flows through the stream and commits via the standalone handling,
+ *   preserving the "commit only what the stream has seen" barrier property.
+ *
+ * See docs/documentdb/documentdb-lsn-sentinel-checkpoints.md for the full design.
+ */
+export class SentinelCheckpointImplementation implements CheckpointImplementation {
+  readonly zeroLsn = SentinelLSN.ZERO.comparable;
+
+  /**
+   * The highest global sentinel value proven so far, merged monotonically from
+   * the resume seed, standalone events and embedded barrier values. Needed
+   * for lsnFromResumeToken: per-batch resume markers carry only a resume token,
+   * so they need the latest observed coordinate paired in.
+   */
+  private position = SentinelLSN.ZERO.sentinel;
+
+  constructor(private context: CheckpointImplementationContext) {}
+
+  parseResumePosition(lsn: string): StreamResumePosition {
+    const parsed = SentinelLSN.fromSerialized(lsn);
+    return { resumeAfter: parsed.resumeToken ?? null, startAfter: null };
+  }
+
+  seedPosition(lsn: string | null): void {
+    this.position = lsn == null ? SentinelLSN.ZERO.sentinel : SentinelLSN.fromSerialized(lsn).sentinel;
+  }
+
+  logResume(lsn: string): void {
+    const parsed = SentinelLSN.fromSerialized(lsn);
+    this.context.logger.info(`Resume streaming at sentinel ${parsed.sentinel} / ${parsed}`);
+  }
+
+  async createStandaloneCheckpoint(): Promise<string> {
+    return createSentinelCheckpointLsn(this.context.client, this.context.db);
+  }
+
+  async createBatchCheckpoint(): Promise<string> {
+    // Advance the shared sentinel checkpoint — the global source-database
+    // coordinate. It must be shared so the LSN domain survives new
+    // ChangeStream instances and new sync rules.
+    // This advance is associated with the change stream in order to track stream
+    // barriers. The returned LSN (counter only, no resume token) is the barrier
+    // marker matched by resolvesBarrier.
+    return createSentinelCheckpointLsn(this.context.client, this.context.db, this.context.checkpointStreamId);
+  }
+
+  async createFirstBarrier(): Promise<string | null> {
+    // The barrier marker is an opaque content-matched marker, not a resume
+    // position, so there is no LSN to open the stream from: a fresh DocumentDB
+    // stream opens from "now" and the snapshot loop re-creates barriers until
+    // one is observed.
+    await this.createBatchCheckpoint();
+    return null;
+  }
+
+  async keepalive(_batch: storage.BucketStorageBatch, _resumeToken: mongo.ResumeToken): Promise<void> {
+    // Advance the shared sentinel, but do not persist a checkpoint here. The
+    // bump is stamped with this stream's id, so its own change event flows
+    // through the stream as an own-barrier event and is committed by the
+    // own-barrier handling, which advances the ordered LSN prefix and refreshes
+    // the resume token (using the event's own token).
+    //
+    // Why bump at all: with an unchanged sentinel, LSN comparison against the
+    // previously persisted LSN falls to the opaque base64 token suffix, which
+    // is not lexicographically meaningful — storage would silently reject
+    // roughly half of plain token refreshes, leaving the persisted resume
+    // token to go stale on an idle stream. (The timestamp implementation avoids this
+    // because its keepalive timestamp is parsed from the resume token itself,
+    // so the ordered prefix always advances with the token.)
+    //
+    // Why not persist immediately: writes that landed after this empty batch
+    // was read — including a write checkpoint head — would be covered by the
+    // persisted LSN before the stream has processed them, allowing write
+    // checkpoints to resolve before the corresponding data is replicated.
+    // Committing only when the bump's event is observed preserves the
+    // "commit only what the stream has seen" barrier property.
+    await createSentinelCheckpointLsn(this.context.client, this.context.db, this.context.checkpointStreamId);
+    this.context.logger.info(
+      `Idle change stream (sentinel implementation). Bumped sentinel to advance the checkpoint.`
+    );
+  }
+
+  lsnFromResumeToken(resumeToken: mongo.ResumeToken) {
+    // Pair the bare token with the current coordinate. The coordinate is
+    // frozen between checkpoint events, so a per-batch resume marker only
+    // advances the token; that is all resumption needs (see the design doc).
+    return { lsn: new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable, timestamp: null };
+  }
+
+  async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
+    const head = await createSentinelCheckpointLsn(this.context.client, this.context.db);
+    const { response, shouldAdvance } = await callback(head);
+    if (shouldAdvance) {
+      // Create another bump to ensure movement after the reported head. This
+      // covers the race where the head's own event is committed before the
+      // write checkpoint document is stored.
+      // Note that this checkpoint should not be associated with a change stream Id.
+      await createSentinelCheckpointLsn(this.context.client, this.context.db);
+    }
+    return response;
+  }
+
+  readonly event: CheckpointEventApi = {
+    observe: (doc) => {
+      const checkpointId = getCheckpointId(doc);
+      if (checkpointId != SENTINEL_CHECKPOINT_ID) {
+        // The sentinel implementation only uses the SENTINEL_CHECKPOINT_ID
+        // document; anything else (including the timestamp impl's standalone
+        // document) is foreign.
+        return 'foreign';
+      }
+      const fullDoc = deserializeFullDocument(doc);
+      if (fullDoc == null) {
+        // An insert/update/replace on our sentinel document must carry a
+        // post-image. A missing fullDocument means the document was deleted or
+        // replaced out from under us, which destroys the coordinate; invalidate
+        // so replication restarts clean instead of stalling.
+        throw new ChangeStreamInvalidatedError(
+          'Sentinel checkpoint event has no fullDocument — cannot read the sentinel',
+          new Error(`Unexpected ${doc.operationType} event on the sentinel checkpoint document`)
+        );
+      }
+      const streamId = fullDoc.stream_id;
+
+      let kind: CheckpointEventKind;
+      if (streamId == null) {
+        // No stream_id: a standalone bump (write checkpoint head, snapshot
+        // marker). Every stream tracks these as the global coordinate.
+        kind = 'standalone';
+      } else if (this.context.checkpointStreamId.equals(streamId)) {
+        // Stamped with our id: one of our own private batch/keepalive barriers.
+        kind = 'own-barrier';
+      } else {
+        // Another stream's private barrier — ignore.
+        return 'foreign';
+      }
+
+      // Both kinds carry the global coordinate in `i`; keep our position current.
+      this.mergePosition(this.readSentinel(fullDoc));
+      return kind;
+    },
+
+    lsn: (doc) => {
+      const fullDoc = deserializeFullDocument(doc);
+      if (fullDoc == null) {
+        throw new ChangeStreamInvalidatedError(
+          'Sentinel checkpoint event has no fullDocument — cannot read the sentinel',
+          new Error(`Unexpected ${doc.operationType} event on the sentinel checkpoint document`)
+        );
+      }
+
+      // Checkpoint events carry the global coordinate in `i`; pair that exact
+      // coordinate with this event's resume token. Plain data-batch resume
+      // markers use lsnFromResumeToken, which pairs the token with the tracked
+      // position instead.
+      return new SentinelLSN({
+        sentinel: this.readSentinel(fullDoc),
+        resume_token: doc._id
+      }).comparable;
+    },
+
+    resolvesBarrier: (marker, doc) => {
+      const fullDoc = deserializeFullDocument(doc);
+      if (fullDoc == null) {
+        this.context.logger.warn('Checkpoint event missing fullDocument — cannot match sentinel barrier');
+        return false;
+      }
+      const parsed = SentinelLSN.fromSerialized(marker);
+      return this.context.checkpointStreamId.equals(fullDoc.stream_id) && fullDoc.i >= parsed.sentinel;
+    }
+  };
+
+  // The sentinel checkpoint document must survive restarts (hence the $ne
+  // below — the startup cleanup deletes every other checkpoint document). Its
+  // counter is the globally-ordered component of every committed LSN, including
+  // write checkpoint heads. Deleting it would re-seed the counter on the next
+  // upsert, risking moving the LSN coordinate system backwards: new commits
+  // could compare below the persisted last_checkpoint_lsn (so storage rejects
+  // them via checkpointBlocked and the checkpoint stalls), and new write
+  // checkpoint heads could resolve against old, higher committed LSNs before
+  // their data has actually replicated.
+  //
+  // Note: this only protects against our own startup cleanup. The global
+  // LSN coordinate still lives in a user-visible collection, so a consumer
+  // can delete it in their source database. Dropping the whole checkpoints
+  // collection is detected (the streaming loop invalidates the stream on the
+  // collection drop event). Deleting just this document is mitigated by
+  // createSentinelCheckpointLsn seeding re-created counters at the current
+  // epoch seconds, so the coordinate jumps forward instead of resetting
+  // below already-committed LSNs.
+  readonly checkpointClearFilter: mongo.Filter<mongo.Document> = { _id: { $ne: SENTINEL_CHECKPOINT_ID } as any };
+
+  /** Never move the position backwards — replayed or reordered events must not regress the coordinate. */
+  private mergePosition(observed: bigint) {
+    if (observed > this.position) {
+      this.position = observed;
+    }
+  }
+
+  /**
+   * Read the sentinel counter from a sentinel checkpoint post-image.
+   *
+   * The `i` field is required: every sentinel write `$inc`s it. If it is
+   * missing, the document was replaced externally, which destroys the LSN
+   * coordinate system (a re-created counter restarts below already-committed
+   * LSNs). Invalidate the stream so replication restarts from scratch instead
+   * of stalling against a broken coordinate.
+   */
+  private readSentinel(fullDoc: mongo.Document): bigint {
+    if (fullDoc.i == null) {
+      throw new ChangeStreamInvalidatedError(
+        'Sentinel checkpoint document has no `i` field — cannot read the sentinel',
+        // JSONBig.stringify, since the post-image is deserialized with useBigInt64.
+        new Error(`Sentinel checkpoint document: ${JSONBig.stringify(fullDoc)}`)
+      );
+    }
+    return fullDoc.i;
+  }
+}
+
+/**
+ * A change event with the decoded `fullDocument` post-image memoized on it, so
+ * a consumer that reads it more than once — e.g. observe() then resolvesBarrier()
+ * for the same event — decodes the buffer only once. `undefined` until first
+ * decoded; `null` when there is no fullDocument.
+ */
+type MemoizedChangeStreamDocument = ProjectedChangeStreamDocument & {
+  parsedFullDocument?: mongo.Document | null;
+};
+
+function deserializeFullDocument(doc: ProjectedChangeStreamDocument): mongo.Document | null {
+  const memo = doc as MemoizedChangeStreamDocument;
+  if (memo.parsedFullDocument !== undefined) {
+    return memo.parsedFullDocument;
+  }
+  const fullDocument = 'fullDocument' in doc ? doc.fullDocument : null;
+  // fullDocument is a raw BSON Buffer from parseChangeDocument.
+  const parsed = fullDocument ? mongo.BSON.deserialize(fullDocument as Buffer, { useBigInt64: true }) : null;
+  memo.parsedFullDocument = parsed;
+  return parsed;
+}

package/src/replication/checkpoints/TimestampCheckpointImplementation.ts

@@ -0,0 +1,145 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { ReplicationHeadCallback, storage } from '@powersync/service-core';
+import { MongoLSN } from '../../common/MongoLSN.js';
+import { createCheckpoint, SENTINEL_CHECKPOINT_ID, STANDALONE_CHECKPOINT_ID } from '../MongoRelation.js';
+import { CHECKPOINTS_COLLECTION, timestampToDate } from '../replication-utils.js';
+import {
+  CheckpointEventApi,
+  CheckpointImplementation,
+  CheckpointImplementationContext,
+  getCheckpointId,
+  getEventTimestamp,
+  StreamResumePosition
+} from './CheckpointImplementation.js';
+
+/**
+ * Standard MongoDB checkpoint implementation. The ordered LSN coordinate is the oplog
+ * clusterTime — unique per operation, monotonic, and parseable from resume
+ * tokens. Barriers and event LSNs are plain comparable LSN strings.
+ */
+export class TimestampCheckpointImplementation implements CheckpointImplementation {
+  readonly zeroLsn = MongoLSN.ZERO.comparable;
+
+  constructor(private context: CheckpointImplementationContext) {}
+
+  parseResumePosition(lsn: string): StreamResumePosition {
+    const parsed = MongoLSN.fromSerialized(lsn);
+    return { resumeAfter: parsed.resumeToken ?? null, startAfter: parsed.timestamp };
+  }
+
+  seedPosition(_lsn: string | null): void {
+    // The coordinate comes from each event's clusterTime; no state to seed.
+  }
+
+  logResume(lsn: string): void {
+    const parsed = MongoLSN.fromSerialized(lsn);
+    // It is normal for this to be a minute or two old when there is a low volume
+    // of ChangeStream events.
+    const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(parsed.timestamp).getTime()) / 1000);
+    this.context.logger.info(
+      `Resume streaming at ${parsed.timestamp.inspect()} / ${parsed} | Token age: ${tokenAgeSeconds}s`
+    );
+  }
+
+  async createStandaloneCheckpoint(): Promise<string> {
+    return createCheckpoint(this.context.db, STANDALONE_CHECKPOINT_ID);
+  }
+
+  async createBatchCheckpoint(): Promise<string> {
+    return createCheckpoint(this.context.db, this.context.checkpointStreamId);
+  }
+
+  async createFirstBarrier(): Promise<string | null> {
+    // The barrier marker is a comparable LSN; resume the snapshot stream from it.
+    return this.createBatchCheckpoint();
+  }
+
+  async keepalive(batch: storage.BucketStorageBatch, resumeToken: mongo.ResumeToken): Promise<void> {
+    // Parse the timestamp from the resume token. The ordered LSN prefix
+    // advances together with the token, so persisting is always safe.
+    const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(resumeToken);
+    await batch.keepalive(lsn);
+    // Log the token update. This helps as a general "replication is still active" message in the logs.
+    // This token would typically be around 10s behind.
+    this.context.logger.info(
+      `Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`
+    );
+  }
+
+  lsnFromResumeToken(resumeToken: mongo.ResumeToken) {
+    // The timestamp is embedded in the resume token.
+    const lsn = MongoLSN.fromResumeToken(resumeToken);
+    return { lsn: lsn.comparable, timestamp: timestampToDate(lsn.timestamp) };
+  }
+
+  async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
+    const session = this.context.client.startSession();
+    try {
+      await this.context.db.command({ hello: 1 }, { session });
+      const head = session.clusterTime?.clusterTime;
+      if (head == null) {
+        throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
+      }
+
+      const { response, shouldAdvance } = await callback(new MongoLSN({ timestamp: head }).comparable);
+
+      if (shouldAdvance) {
+        // Trigger a change on the changestream, so that the write checkpoint
+        // is processed without waiting for other writes.
+        await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
+          {
+            _id: STANDALONE_CHECKPOINT_ID as any
+          },
+          {
+            $inc: { i: 1 }
+          },
+          {
+            upsert: true,
+            returnDocument: 'after',
+            session
+          }
+        );
+        const time = session.operationTime!;
+        if (time == null) {
+          throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
+        } else if (time.lt(head)) {
+          throw new ServiceAssertionError(`operationTime must be > clusterTime`);
+        }
+      }
+
+      return response;
+    } finally {
+      await session.endSession();
+    }
+  }
+
+  readonly event: CheckpointEventApi = {
+    observe: (doc) => {
+      const checkpointId = getCheckpointId(doc);
+      if (checkpointId == null || checkpointId == SENTINEL_CHECKPOINT_ID) {
+        return 'foreign';
+      }
+      // The STANDALONE_CHECKPOINT_ID is only used for the TimestampCheckpointImplementation
+      if (checkpointId == STANDALONE_CHECKPOINT_ID) {
+        return 'standalone';
+      }
+      return this.context.checkpointStreamId.equals(checkpointId) ? 'own-barrier' : 'foreign';
+    },
+
+    lsn: (doc) => {
+      return new MongoLSN({
+        timestamp: getEventTimestamp(doc),
+        resume_token: doc._id
+      }).comparable;
+    },
+
+    resolvesBarrier: (marker, doc) => {
+      // Barrier markers are comparable LSNs in this implementation.
+      return this.event.lsn(doc) >= marker;
+    }
+  };
+
+  // It's safe to clear the entire _powersync_checkpoints collection in this mode.
+  readonly checkpointClearFilter: mongo.Filter<mongo.Document> = {};
+}

package/src/replication/checkpoints/create-checkpoint-implementation.ts

@@ -0,0 +1,14 @@
+import { CheckpointImplementation, CheckpointImplementationContext } from './CheckpointImplementation.js';
+import { SentinelCheckpointImplementation } from './SentinelCheckpointImplementation.js';
+import { TimestampCheckpointImplementation } from './TimestampCheckpointImplementation.js';
+
+/**
+ * Select the checkpoint implementation for a source: sentinel-based for DocumentDB
+ * DB, clusterTime-based for standard MongoDB.
+ */
+export function createCheckpointImplementation(
+  isDocumentDb: boolean,
+  context: CheckpointImplementationContext
+): CheckpointImplementation {
+  return isDocumentDb ? new SentinelCheckpointImplementation(context) : new TimestampCheckpointImplementation(context);
+}

package/src/replication/replication-utils.ts

@@ -1,3 +1,4 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as bson from 'bson';
 import { PostImagesOption } from '../types/types.js';
@@ -7,16 +8,36 @@ export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
 
 const REQUIRED_CHECKPOINT_PERMISSIONS = ['find', 'insert', 'update', 'remove', 'changeStream', 'createCollection'];
 
+/**
+ * Whether a `hello` response indicates Azure DocumentDB (formerly Azure Cosmos
+ * DB for MongoDB vCore), which reports `documentdb_versions` in the `internal`
+ * section.
+ */
+function isDocumentDbHello(hello: mongo.Document): boolean {
+  return hello.internal?.documentdb_versions != null;
+}
+
+/**
+ * Detect whether the connected server is DocumentDB. DocumentDB lacks usable
+ * clusterTime/operationTime and uses the sentinel checkpoint implementation.
+ */
+export async function detectDocumentDb(db: mongo.Db): Promise<boolean> {
+  const hello = await db.command({ hello: 1 });
+  return isDocumentDbHello(hello);
+}
+
 export async function checkSourceConfiguration(connectionManager: MongoManager): Promise<void> {
   const db = connectionManager.db;
 
   const hello = await db.command({ hello: 1 });
-  if (hello.msg == 'isdbgrid') {
+  const isDocumentDb = isDocumentDbHello(hello);
+
+  if (hello.msg == 'isdbgrid' && !isDocumentDb) {
     throw new ServiceError(
       ErrorCode.PSYNC_S1341,
       'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
     );
-  } else if (hello.setName == null) {
+  } else if (hello.setName == null && !isDocumentDb) {
     throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
   }
 

package/src/types/types.ts

@@ -1,5 +1,6 @@
 import type { MongoConnectionParams } from '@powersync/lib-service-mongodb/types';
 import * as lib_mongo from '@powersync/lib-service-mongodb/types';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as service_types from '@powersync/service-types';
 import { LookupFunction } from 'node:net';
 import * as t from 'ts-codec';
@@ -40,6 +41,10 @@ export enum PostImagesOption {
   READ_ONLY = 'read_only'
 }
 
+const DEFAULT_PING_INTERVAL_SECONDS = 60;
+const MIN_PING_INTERVAL_SECONDS = 5;
+const MAX_PING_INTERVAL_SECONDS = 60;
+
 export interface NormalizedMongoConnectionConfig {
   id: string;
   tag: string;
@@ -54,13 +59,19 @@ export interface NormalizedMongoConnectionConfig {
 
   postImages: PostImagesOption;
 
+  heartbeat_interval_seconds: number;
+
   connectionParams: MongoConnectionParams;
 }
 
 export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(
   t.object({
     // Replication specific settings
-    post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
+    post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional(),
+    /**
+     * Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
+     */
+    heartbeat_interval_seconds: t.number.or(t.Null).optional()
   })
 );
 
@@ -87,6 +98,20 @@ export function normalizeConnectionConfig(options: MongoConnectionConfigDecoded)
     ...base,
     id: options.id ?? 'default',
     tag: options.tag ?? 'default',
-    postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF
+    postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF,
+    heartbeat_interval_seconds: normalizeHeartbeatInterval(options.heartbeat_interval_seconds)
   };
 }
+
+function normalizeHeartbeatInterval(value: number | null | undefined): number {
+  if (value == null) {
+    return DEFAULT_PING_INTERVAL_SECONDS;
+  }
+  if (!Number.isFinite(value) || value < MIN_PING_INTERVAL_SECONDS || value > MAX_PING_INTERVAL_SECONDS) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1109,
+      `MongoDB connection: heartbeat_interval_seconds must be between ${MIN_PING_INTERVAL_SECONDS} and ${MAX_PING_INTERVAL_SECONDS} seconds; null or omitted defaults to ${DEFAULT_PING_INTERVAL_SECONDS}`
+    );
+  }
+  return value;
+}

package/test/DOCUMENTDB_TESTING.md

@@ -0,0 +1,115 @@
+# Running Tests Against DocumentDB
+
+These instructions cover running the `module-mongodb` test suite against an Azure DocumentDB (formerly Cosmos DB for MongoDB vCore) cluster.
+
+## Prerequisites
+
+- An Azure DocumentDB (Cosmos DB for MongoDB vCore) cluster with change stream support
+- Local PostgreSQL for PowerSync's internal storage (not the source database)
+- The connection URI for the DocumentDB cluster
+
+> **The open-source `documentdb-local` Docker image cannot be used for these tests.** The
+> open-source DocumentDB engine does not implement change streams (`$changeStream is not
+supported yet in native pipeline`), does not report the `documentdb_versions` `hello` field
+> the suite uses for detection, and presents as `msg: isdbgrid`. An Azure-managed DocumentDB
+> (vCore) cluster is required.
+
+## Environment Variables
+
+DocumentDB is detected automatically from the server: the test suite runs
+`detectDocumentDb()` once at startup (see `DatabaseType.ts`) and gates the
+DocumentDB-specific tests on the result. There is no separate enable flag — pointing
+`MONGO_TEST_DATA_URL` at a DocumentDB cluster is what activates the DocumentDB tests.
+
+| Variable                  | Required | Description                                                                                                                                                              |
+| ------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `MONGO_TEST_DATA_URL`     | Yes      | DocumentDB connection URI. Must include a database name in the path (see below). Pointing this at a DocumentDB cluster enables the DocumentDB tests.                     |
+| `PG_STORAGE_TEST_URL`     | No       | PostgreSQL connection for PowerSync storage. Defaults to `postgres://postgres:postgres@localhost:5432/powersync_storage_test`.                                           |
+| `TEST_MONGO_STORAGE`      | No       | Set to `false` to skip MongoDB storage tests. Recommended when testing against DocumentDB to avoid using it as a storage backend.                                        |
+| `TEST_TIMEOUT_MULTIPLIER` | No       | Scale factor `testTimeout()` applies to a default timeout on DocumentDB when no explicit cloud override is given. Defaults to **6**. Increase for a slow/remote cluster. |
+
+### Connection URI format
+
+The `MONGO_TEST_DATA_URL` must include a database name in the path. DocumentDB URIs typically don't have one, so you need to add it before the query string:
+
+```
+# Original URI (no database):
+mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/
+
+# With database added:
+mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/powersync_test
+```
+
+If your password contains special characters (`=`, `@`, `+`, `/`), they must be URL-encoded in the URI (e.g., `=` becomes `%3D`). DocumentDB auto-generated passwords often contain `=` (base64).
+
+## Commands
+
+All commands run from the module directory: `modules/module-mongodb/`
+
+```bash
+# Run all DocumentDB tests (integration + unit helpers):
+MONGO_TEST_DATA_URL="mongodb+srv://user:pass@cluster.mongocluster.cosmos.azure.com/powersync_test" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb --reporter=verbose
+
+# Run only integration tests:
+MONGO_TEST_DATA_URL="<uri>" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb_mode --reporter=verbose
+
+# Run only unit helper tests (no DocumentDB cluster needed):
+npx vitest run documentdb_helpers --reporter=verbose
+
+# Run a specific test by name:
+MONGO_TEST_DATA_URL="<uri>" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb_mode -t "resume after restart" --reporter=verbose
+```
+
+If you have the URI in an environment variable (e.g., `$DOCUMENTDB_URI`), you can construct the test URL inline:
+
+```bash
+DOCUMENTDB_TEST_URL=$(echo "$DOCUMENTDB_URI" | sed 's|\?|powersync_test?|')
+MONGO_TEST_DATA_URL="$DOCUMENTDB_TEST_URL" \
+TEST_MONGO_STORAGE=false \
+npx vitest run documentdb --reporter=verbose
+```
+
+## GitHub Actions
+
+The `.github/workflows/documentdb-integration.yml` workflow runs these tests manually via `workflow_dispatch` only. Add a repository or organization secret named `AZURE_DOCUMENTDB_TEST_DATA_URL`, then dispatch the workflow. The workflow maps that secret to the test suite's `MONGO_TEST_DATA_URL` environment variable.
+
+The URI must include a database name in the path. The tests clear/drop this database as part of setup, so use a dedicated test database and cluster.
+
+The workflow starts a local MongoDB storage service, then runs the complete `modules/module-mongodb` `test` script against that storage backend. `MONGO_TEST_DATA_URL` is the remote DocumentDB source; `MONGO_TEST_URL` points at the local MongoDB storage test instance.
+
+```bash
+TEST_MONGO_STORAGE=true
+TEST_POSTGRES_STORAGE=false
+```
+
+## Test Files
+
+| File                         | Requires DocumentDB       | Description                                                                                                                                                                    |
+| ---------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `documentdb_mode.test.ts`    | Yes                       | Integration tests: replication, sentinel checkpoints, write checkpoints, keepalive, resume. Skipped automatically unless `MONGO_TEST_DATA_URL` points at a DocumentDB cluster. |
+| `documentdb_helpers.test.ts` | No (1 test needs MongoDB) | Unit tests: `getEventTimestamp`, sentinel parsing/matching, detection logic. Runs against any MongoDB or standalone.                                                           |
+
+## What the Integration Tests Cover
+
+Each integration test runs against 3 storage versions (v1, v2, v3) = 15 integration tests. Plus 15 unit tests in helpers = 30 total.
+
+| Test                    | What it validates                                                                               |
+| ----------------------- | ----------------------------------------------------------------------------------------------- |
+| basic replication       | Insert, update, delete through change stream with wallTime timestamps                           |
+| sentinel checkpoint     | Checkpoint created with `mode: 'sentinel'`, resolved by matching document content in the stream |
+| keepalive               | Stream idles past the keepalive interval without crashing on DocumentDB resume tokens           |
+| write checkpoint        | Full `createReplicationHead` → sentinel → polling flow for client write consistency             |
+| data events not dropped | Verifies `.lte()` dedup guard is skipped — events in the same wall-clock second are not lost    |
+| resume after restart    | Stop streaming, create new context, resume from stored token                                    |
+
+There is also a **characterization test**, `does not report collection drop and rename events`,
+that documents the current limitation: it writes DDL plus a post-DDL marker, waits for the marker
+(proving the stream caught up), then asserts no `drop` / `rename` events were delivered. It passes
+today and **fails if a future DocumentDB engine starts delivering DDL events** — a signal to add
+real drop/rename replication support.

package/test/src/DatabaseType.ts

@@ -0,0 +1,25 @@
+import { detectDocumentDb } from '@module/replication/replication-utils.js';
+import { logger } from '@powersync/lib-services-framework';
+import { connectMongoData } from './util.js';
+
+export enum DatabaseType {
+  DOCUMENTDB = 'DOCUMENTDB',
+  MONGODB = 'MONGODB'
+}
+
+let _databaseType: DatabaseType = DatabaseType.MONGODB;
+
+// Detected once at import time. Close the client afterwards so this detection
+// does not leak a connection: this module is imported by many test files, and
+// the client created here is otherwise never closed, accumulating open handles
+// in Vitest.
+const { client, db } = await connectMongoData();
+try {
+  _databaseType = (await detectDocumentDb(db)) ? DatabaseType.DOCUMENTDB : DatabaseType.MONGODB;
+} catch (ex) {
+  logger.warn(`Could not determine MongoDB database type`, ex);
+} finally {
+  await client.close().catch(() => {});
+}
+
+export const DATABASE_TYPE = _databaseType;