@powersync/service-module-mongodb 0.18.1 → 0.19.0

package/test/src/documentdb_mode.test.ts

@@ -0,0 +1,1040 @@
+import { setTimeout } from 'node:timers/promises';
+import { describe, expect, test, vi } from 'vitest';
+
+import { createWriteCheckpoint, WriteCheckpointBatcher } from '@powersync/service-core';
+import { test_utils } from '@powersync/service-core-tests';
+
+import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
+import { SentinelLSN } from '@module/common/SentinelLSN.js';
+import { createSentinelCheckpointLsn, SENTINEL_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
+import { CHECKPOINTS_COLLECTION } from '@module/replication/replication-utils.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { env } from './env.js';
+import { testTimeout } from './test-timeouts.js';
+import { connectMongoData, describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data"
+`;
+
+const DOCUMENTDB_BUCKET_DATA_TIMEOUT = testTimeout(20_000, { cloudOverride: 120_000 });
+const DOCUMENTDB_LARGE_BUCKET_DATA_TIMEOUT = testTimeout(60_000, { cloudOverride: 180_000 });
+const DOCUMENTDB_LARGE_DOCUMENT_TIMEOUT = testTimeout(120_000, { cloudOverride: 360_000 });
+const DOCUMENTDB_LARGE_DOCUMENT_MAX_AWAIT_TIME_MS = testTimeout(10_000, { cloudOverride: 30_000 });
+const DOCUMENTDB_RESTART_TIMEOUT = testTimeout(50_000, { cloudOverride: 120_000 });
+
+// These tests require a real DocumentDB cluster. See test/DOCUMENTDB_TESTING.md for setup.
+//
+// Why these can't run against standard MongoDB: the DocumentDB workarounds involve
+// different change stream initialization ordering (lazy ChangeStream + no
+// startAtOperationTime) and wall-clock LSN precision (increment 0 instead of
+// operationTime's real increments). These produce LSN comparison failures when
+// mixed with standard MongoDB's operationTime-based checkpoints. A test flag that
+// partially simulates DocumentDB creates more problems than it solves — see the
+// commit history on the cosmos branch for the full investigation.
+describe.skipIf(DATABASE_TYPE != DatabaseType.DOCUMENTDB)('documentDbMode', () => {
+  test('prints hello response and detects DocumentDB', async () => {
+    const { client, db } = await connectMongoData();
+    try {
+      const hello = await db.command({ hello: 1 });
+      console.dir({ hello }, { depth: null });
+      expect(hello.internal?.documentdb_versions != null).toBe(true);
+    } finally {
+      await client.close();
+    }
+  });
+
+  // Verifies a core assumption of the DocumentDB sentinel LSN design: that
+  // `fullDocument` on update events is the write-time post-image, not a
+  // read-time lookup of the document's current state.
+  //
+  // SentinelCheckpointImplementation tracks its position from `fullDocument.i` on
+  // updates to the sentinel checkpoint document, and commits LSNs based on
+  // that value. With read-time (updateLookup) semantics, a backlogged stream
+  // reading an old event would see a *future* counter value, letting the
+  // committed LSN run ahead of unprocessed data events — which would allow
+  // write checkpoints to resolve before the caller's write has replicated.
+  //
+  // The test bursts increments while no cursor is reading, then replays the
+  // events. Write-time semantics yield one event per increment, each carrying
+  // its own value. Read-time semantics yield events all reporting the final
+  // value. Collapsed/missing events would indicate DocumentDB coalesces rapid
+  // same-document updates, which would be its own significant finding.
+  test('fullDocument on update events is the write-time post-image', { timeout: 120_000 }, async () => {
+    const { client, db } = await connectMongoData();
+    try {
+      const collection = db.collection<{ _id: string; i: number }>(CHECKPOINTS_COLLECTION);
+      const docId = '_test_fulldoc_semantics';
+
+      const pipeline = [
+        {
+          $match: {
+            operationType: 'update',
+            'ns.db': db.databaseName,
+            'ns.coll': CHECKPOINTS_COLLECTION,
+            'documentKey._id': docId
+          }
+        }
+      ];
+
+      // Establish a resume position past the priming writes, so the replay
+      // below starts cleanly before the burst.
+      //
+      // The stream only delivers events written after the cursor is
+      // established (DocumentDB has no startAtOperationTime), and the driver opens
+      // the cursor lazily on the first read. So keep writing priming
+      // increments until the first event comes through — the same approach the
+      // production retry loop uses during initial LSN acquisition.
+      let resumeToken: unknown;
+      const before = client.watch(pipeline, { fullDocument: 'updateLookup', maxAwaitTimeMS: 500 });
+      try {
+        const deadline = Date.now() + 60_000;
+        let lastPrimedI = 0;
+        let drainedTo = -1;
+        while (drainedTo < 0 && Date.now() < deadline) {
+          const result = await collection.findOneAndUpdate(
+            { _id: docId },
+            { $inc: { i: 1 } },
+            { upsert: true, returnDocument: 'after' }
+          );
+          lastPrimedI = result!.i;
+          const event = await before.tryNext();
+          if (event != null && 'fullDocument' in event) {
+            drainedTo = event.fullDocument?.i ?? -1;
+          }
+        }
+        expect(drainedTo, 'change stream never delivered the priming events').toBeGreaterThan(0);
+
+        // Drain the remaining priming events so they don't leak into the
+        // replay below.
+        while (drainedTo < lastPrimedI && Date.now() < deadline) {
+          const event = await before.tryNext();
+          if (event != null && 'fullDocument' in event) {
+            drainedTo = event.fullDocument?.i ?? drainedTo;
+          }
+        }
+        expect(drainedTo, 'timed out draining priming events').toEqual(lastPrimedI);
+
+        resumeToken = before.resumeToken;
+        expect(resumeToken).toBeTruthy();
+      } finally {
+        await before.close();
+      }
+
+      // Burst increments with no cursor reading. All writes complete before
+      // any event is fetched, so a read-time lookup cannot accidentally
+      // return event-time values.
+      const expected: number[] = [];
+      for (let k = 0; k < 5; k++) {
+        const result = await collection.findOneAndUpdate(
+          { _id: docId },
+          { $inc: { i: 1 } },
+          { upsert: true, returnDocument: 'after' }
+        );
+        expected.push(result!.i);
+      }
+
+      // Replay the burst and record the value each event carries.
+      const seen: number[] = [];
+      const after = client.watch(pipeline, {
+        fullDocument: 'updateLookup',
+        maxAwaitTimeMS: 500,
+        resumeAfter: resumeToken
+      });
+      try {
+        const start = Date.now();
+        const deadline = start + 60_000;
+        while (seen.length < expected.length && Date.now() < deadline) {
+          const event = await after.tryNext();
+          if (event != null && 'fullDocument' in event) {
+            seen.push(event.fullDocument?.i);
+          }
+        }
+      } finally {
+        await after.close();
+      }
+
+      expect(seen).toEqual(expected);
+    } finally {
+      await client.close();
+    }
+  });
+
+  // Empirically probe whether DocumentDB delivers change events for *different*
+  // documents in the order they were written.
+  //
+  // DocumentDB only guarantees change order per shard key (RU docs) / nothing
+  // documented (vCore) — not globally. The sentinel write-checkpoint design
+  // assumes no cross-document order: a write checkpoint head is the
+  // `_sentinel_checkpoint` counter, and the caller's data write lives in a
+  // different document. If the sentinel event can be delivered *before* the
+  // data write that preceded it, a write checkpoint can resolve before its data
+  // has replicated (see docs/documentdb/documentdb-lsn-sentinel-checkpoints.md#change-ordering).
+  //
+  // This reproduces that exact scenario using the real checkpointing write path
+  // (createSentinelCheckpointLsn): each round writes a data document, then bumps
+  // the sentinel counter, and we record the delivery order off a single
+  // cluster-level change stream.
+  //
+  // We do NOT assert on cross-document order (delivery order is not contractual
+  // — a single-shard cluster may happen to preserve it); the finding is logged
+  // so we can document the cluster's observed behaviour. We DO assert that
+  // same-document order is preserved, since sentinel `i` matching relies on it.
+  test.skipIf(!env.SLOW_TESTS)('change stream delivery order across documents', { timeout: 30_000 }, async () => {
+    const { client, db } = await connectMongoData();
+    const ROUNDS = 20;
+    const dataColl = 'order_probe_data';
+    const dataId = '_order_probe_doc';
+    try {
+      const data = db.collection<{ _id: string; seq: number }>(dataColl);
+      await data.deleteMany({});
+
+      // Watch the whole database; classify events by documentKey in code.
+      const pipeline = [
+        {
+          $match: {
+            operationType: { $in: ['insert', 'update', 'replace'] },
+            'ns.db': db.databaseName
+          }
+        }
+      ];
+
+      const cursor = client.watch(pipeline, { fullDocument: 'updateLookup', maxAwaitTimeMS: 500 });
+      try {
+        // Prime the cursor (DocumentDB has no startAtOperationTime, and the driver
+        // opens the cursor lazily) by writing the data doc with descending
+        // negative seqs until an event comes through, then drain the backlog.
+        const primeDeadline = Date.now() + 60_000;
+        let primed = false;
+        let primeSeq = 0;
+        while (!primed && Date.now() < primeDeadline) {
+          primeSeq -= 1;
+          await data.updateOne({ _id: dataId }, { $set: { seq: primeSeq } }, { upsert: true });
+          if ((await cursor.tryNext()) != null) {
+            primed = true;
+          }
+        }
+        expect(primed, 'change stream cursor never went live').toBe(true);
+        while ((await cursor.tryNext()) != null) {
+          // drain remaining priming events
+        }
+
+        // Issue interleaved writes: data(round) then sentinel bump, per round.
+        type Issued = { label: string; kind: 'data' | 'sentinel'; round: number; sentinel?: bigint };
+        const issued: Issued[] = [];
+        const sentinelValueToRound = new Map<string, number>();
+        for (let round = 1; round <= ROUNDS; round++) {
+          await data.updateOne({ _id: dataId }, { $set: { seq: round } });
+          issued.push({ label: `D${round}`, kind: 'data', round });
+
+          const sentinel = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(client, db)).sentinel;
+          issued.push({ label: `S${round}`, kind: 'sentinel', round, sentinel });
+          sentinelValueToRound.set(sentinel.toString(), round);
+        }
+
+        // Collect the delivered events in arrival order.
+        const arrivals: { label: string; kind: 'data' | 'sentinel'; value: number | bigint }[] = [];
+        const collectDeadline = Date.now() + 90_000;
+        while (arrivals.length < issued.length && Date.now() < collectDeadline) {
+          const ev = await cursor.tryNext();
+          if (ev == null || !('documentKey' in ev) || !('fullDocument' in ev) || ev.fullDocument == null) {
+            continue;
+          }
+          const id = String(ev.documentKey._id);
+          if (id === dataId) {
+            const seq = (ev.fullDocument as any).seq as number;
+            if (seq >= 1) {
+              arrivals.push({ label: `D${seq}`, kind: 'data', value: seq });
+            }
+          } else if (id === SENTINEL_CHECKPOINT_ID) {
+            const value = BigInt((ev.fullDocument as any).i);
+            const round = sentinelValueToRound.get(value.toString());
+            if (round != null) {
+              arrivals.push({ label: `S${round}`, kind: 'sentinel', value });
+            }
+          }
+        }
+
+        // --- Analysis ---
+        const dataArrivals = arrivals.filter((a) => a.kind === 'data').map((a) => a.value as number);
+        const sentinelArrivals = arrivals.filter((a) => a.kind === 'sentinel').map((a) => a.value as bigint);
+        const dataOrderPreserved = dataArrivals.every((v, i) => i === 0 || v > dataArrivals[i - 1]);
+        const sentinelOrderPreserved = sentinelArrivals.every((v, i) => i === 0 || v > sentinelArrivals[i - 1]);
+
+        // General cross-document inversions: any event delivered out of global issue order.
+        const issueIndex = new Map(issued.map((it, i) => [it.label, i] as const));
+        const crossDocumentInversions: { afterDelivering: string; received: string }[] = [];
+        for (let i = 1; i < arrivals.length; i++) {
+          if (issueIndex.get(arrivals[i].label)! < issueIndex.get(arrivals[i - 1].label)!) {
+            crossDocumentInversions.push({ afterDelivering: arrivals[i - 1].label, received: arrivals[i].label });
+          }
+        }
+
+        // Bug-specific signal: a sentinel bump overtaking the data write that
+        // preceded it in the same round (write checkpoint resolves too early).
+        const sentinelOvertakingItsDataWrite: number[] = [];
+        for (let round = 1; round <= ROUNDS; round++) {
+          const dIdx = arrivals.findIndex((a) => a.label === `D${round}`);
+          const sIdx = arrivals.findIndex((a) => a.label === `S${round}`);
+          if (dIdx >= 0 && sIdx >= 0 && sIdx < dIdx) {
+            sentinelOvertakingItsDataWrite.push(round);
+          }
+        }
+
+        console.dir(
+          {
+            crossDocumentOrderingProbe: {
+              rounds: ROUNDS,
+              issued: issued.length,
+              arrived: arrivals.length,
+              deliveryOrder: arrivals.map((a) => a.label),
+              dataOrderPreserved,
+              sentinelOrderPreserved,
+              crossDocumentInversions,
+              sentinelOvertakingItsDataWrite
+            }
+          },
+          { depth: null }
+        );
+
+        // We expect to receive every issued event within the deadline.
+        expect(arrivals.length, 'did not receive all issued events before the deadline').toBe(issued.length);
+        // Per-document ordering is the one guarantee DocumentDB documents — assert it.
+        expect(dataOrderPreserved, 'data-document events arrived out of order').toBe(true);
+        expect(sentinelOrderPreserved, 'sentinel-document events arrived out of order').toBe(true);
+        // Cross-document order is intentionally NOT asserted; inversions (if any)
+        // are logged above as the finding.
+      } finally {
+        await cursor.close();
+      }
+      await data.deleteMany({});
+    } finally {
+      await client.close();
+    }
+  });
+
+  // Follow-up to the probe above. The sequential probe preserves order on a
+  // single-shard cluster, where there is effectively one ordered feed. This
+  // variant adds a concurrent write storm across many other documents and a much
+  // higher round count, to test whether a single-shard cluster has *internal*
+  // sub-partitions whose feeds can interleave out of order under load.
+  //
+  // The measured D→S pairs are still issued strictly sequentially (so they keep a
+  // well-defined happens-before order to violate); the background writers only
+  // generate feed pressure. Same reporting; cross-document order is logged, not
+  // asserted.
+  test.skipIf(!env.SLOW_TESTS)(
+    'change stream delivery order across documents — under concurrent load',
+    { timeout: 240_000 },
+    async () => {
+      const { client, db } = await connectMongoData();
+      const ROUNDS = 100;
+      const BACKGROUND_WRITERS = 6;
+      const dataColl = 'order_probe_data';
+      const dataId = '_order_probe_doc';
+      const backgroundCollections = Array.from({ length: BACKGROUND_WRITERS }, (_, w) => `order_probe_bg_${w}`);
+      try {
+        const data = db.collection<{ _id: string; seq: number }>(dataColl);
+        await data.deleteMany({});
+
+        // Only deliver the two measured namespaces, so background churn does not
+        // flood the cursor we are measuring.
+        const pipeline = [
+          {
+            $match: {
+              operationType: { $in: ['insert', 'update', 'replace'] },
+              'ns.db': db.databaseName,
+              'ns.coll': { $in: [dataColl, CHECKPOINTS_COLLECTION] }
+            }
+          }
+        ];
+        const cursor = client.watch(pipeline, { fullDocument: 'updateLookup', maxAwaitTimeMS: 500 });
+
+        // Start the background write storm: 6 loops, each with one write in flight,
+        // churning 100 rotating documents per collection.
+        let stopBackground = false;
+        const background = backgroundCollections.map((name) =>
+          (async () => {
+            const coll = db.collection<{ _id: string; n: number }>(name);
+            let n = 0;
+            while (!stopBackground) {
+              n += 1;
+              await coll.updateOne({ _id: `bg_${n % 100}` }, { $set: { n } }, { upsert: true }).catch(() => {});
+            }
+          })()
+        );
+
+        try {
+          // Prime the cursor.
+          const primeDeadline = Date.now() + 60_000;
+          let primed = false;
+          let primeSeq = 0;
+          while (!primed && Date.now() < primeDeadline) {
+            primeSeq -= 1;
+            await data.updateOne({ _id: dataId }, { $set: { seq: primeSeq } }, { upsert: true });
+            if ((await cursor.tryNext()) != null) {
+              primed = true;
+            }
+          }
+          expect(primed, 'change stream cursor never went live').toBe(true);
+          while ((await cursor.tryNext()) != null) {
+            // drain priming events
+          }
+
+          // Measured sequential D→S pairs.
+          const issued: { label: string; round: number }[] = [];
+          const sentinelValueToRound = new Map<string, number>();
+          for (let round = 1; round <= ROUNDS; round++) {
+            await data.updateOne({ _id: dataId }, { $set: { seq: round } });
+            issued.push({ label: `D${round}`, round });
+            const sentinel = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(client, db)).sentinel;
+            issued.push({ label: `S${round}`, round });
+            sentinelValueToRound.set(sentinel.toString(), round);
+          }
+
+          // Collect.
+          const arrivals: string[] = [];
+          const expectedLabels = new Set(issued.map((i) => i.label));
+          const collectDeadline = Date.now() + 120_000;
+          while (arrivals.length < issued.length && Date.now() < collectDeadline) {
+            const ev = await cursor.tryNext();
+            if (ev == null || !('documentKey' in ev) || !('fullDocument' in ev) || ev.fullDocument == null) {
+              continue;
+            }
+            const id = String(ev.documentKey._id);
+            let label: string | null = null;
+            if (id === dataId) {
+              const seq = (ev.fullDocument as any).seq as number;
+              if (seq >= 1) {
+                label = `D${seq}`;
+              }
+            } else if (id === SENTINEL_CHECKPOINT_ID) {
+              const round = sentinelValueToRound.get(BigInt((ev.fullDocument as any).i).toString());
+              if (round != null) {
+                label = `S${round}`;
+              }
+            }
+            if (label != null && expectedLabels.has(label)) {
+              arrivals.push(label);
+            }
+          }
+
+          // Analysis (same as the sequential probe).
+          const issueIndex = new Map(issued.map((it, i) => [it.label, i] as const));
+          const crossDocumentInversions: { afterDelivering: string; received: string }[] = [];
+          for (let i = 1; i < arrivals.length; i++) {
+            if (issueIndex.get(arrivals[i])! < issueIndex.get(arrivals[i - 1])!) {
+              crossDocumentInversions.push({ afterDelivering: arrivals[i - 1], received: arrivals[i] });
+            }
+          }
+          const sentinelOvertakingItsDataWrite: number[] = [];
+          for (let round = 1; round <= ROUNDS; round++) {
+            const dIdx = arrivals.indexOf(`D${round}`);
+            const sIdx = arrivals.indexOf(`S${round}`);
+            if (dIdx >= 0 && sIdx >= 0 && sIdx < dIdx) {
+              sentinelOvertakingItsDataWrite.push(round);
+            }
+          }
+
+          console.dir(
+            {
+              crossDocumentOrderingUnderLoad: {
+                rounds: ROUNDS,
+                backgroundWriters: BACKGROUND_WRITERS,
+                issued: issued.length,
+                arrived: arrivals.length,
+                crossDocumentInversions,
+                sentinelOvertakingItsDataWrite
+              }
+            },
+            { depth: null }
+          );
+
+          expect(arrivals.length, 'did not receive all issued events before the deadline').toBe(issued.length);
+        } finally {
+          stopBackground = true;
+          await Promise.allSettled(background);
+          await cursor.close();
+        }
+
+        await data.deleteMany({});
+        for (const name of backgroundCollections) {
+          await db
+            .collection(name)
+            .drop()
+            .catch(() => {});
+        }
+      } finally {
+        await client.close();
+      }
+    }
+  );
+
+  // Characterize whether DocumentDB's change stream reports collection `drop` and
+  // `rename` DDL events. PowerSync does not currently replicate these on
+  // DocumentDB (see docs/documentdb/documentdb-limitations.md, "Collection drop
+  // and rename are not replicated") — they are delivered differently, or not at
+  // all, through the cluster-level change stream.
+  //
+  // After the DDL, a normal insert (the "marker") is written and we wait for its
+  // event to arrive. That proves the stream is live and has caught up past the
+  // DDL, so the absence of `drop`/`rename` events is genuine, not just
+  // not-waited-long-enough. The test then asserts those events are NOT delivered,
+  // documenting the current limitation. If a future DocumentDB engine starts
+  // delivering them, this test fails — a signal to add real drop/rename support.
+  test('does not report collection drop and rename events', { timeout: 120_000 }, async () => {
+    const { client, db } = await connectMongoData();
+    const dropColl = 'ddl_probe_drop';
+    const renameSrc = 'ddl_probe_rename_src';
+    const renameDst = 'ddl_probe_rename_dst';
+    const markerColl = 'ddl_probe_marker';
+    const markerId = 'ddl_probe_marker_doc';
+    const cleanup = async () => {
+      for (const c of [dropColl, renameSrc, renameDst, markerColl]) {
+        await db
+          .collection(c)
+          .drop()
+          .catch(() => {});
+      }
+    };
+    try {
+      await cleanup();
+
+      // Cluster-level stream (DocumentDB only supports cluster-level), watching
+      // the whole database so DDL events on any collection are visible.
+      const pipeline = [{ $match: { 'ns.db': db.databaseName } }];
+      const cursor = client.watch(pipeline, { fullDocument: 'updateLookup', maxAwaitTimeMS: 500 });
+      try {
+        // Prime the cursor so it is live before the DDL operations.
+        const primeDeadline = Date.now() + 60_000;
+        let primed = false;
+        let n = 0;
+        while (!primed && Date.now() < primeDeadline) {
+          n += 1;
+          await db.collection(renameSrc).updateOne({ _id: 'prime' as any }, { $set: { n } }, { upsert: true });
+          if ((await cursor.tryNext()) != null) {
+            primed = true;
+          }
+        }
+        expect(primed, 'change stream cursor never went live').toBe(true);
+        while ((await cursor.tryNext()) != null) {
+          // drain priming events
+        }
+
+        // Perform the DDL: drop one collection, rename another.
+        await db.collection(dropColl).insertOne({ x: 1 });
+        await db.collection(dropColl).drop();
+        await client.db('admin').command({
+          renameCollection: `${db.databaseName}.${renameSrc}`,
+          to: `${db.databaseName}.${renameDst}`
+        });
+
+        // Marker: a normal insert issued *after* the DDL. When its event is
+        // delivered, the stream has provably caught up past the DDL operations.
+        await db.collection(markerColl).insertOne({ _id: markerId as any });
+
+        // Collect events until the marker event arrives (the "caught up" signal).
+        const seen: string[] = [];
+        let markerSeen = false;
+        const deadline = Date.now() + 30_000;
+        while (!markerSeen && Date.now() < deadline) {
+          const ev = await cursor.tryNext();
+          if (ev == null) {
+            continue;
+          }
+          if (ev.operationType === 'insert' && 'documentKey' in ev && String(ev.documentKey._id) === markerId) {
+            markerSeen = true;
+            break;
+          }
+          seen.push(ev.operationType);
+        }
+        // Drain anything already buffered after the marker, in case a DDL event
+        // was delivered slightly out of order behind it.
+        let trailing: Awaited<ReturnType<typeof cursor.tryNext>>;
+        while ((trailing = await cursor.tryNext()) != null) {
+          seen.push(trailing.operationType);
+        }
+
+        console.dir({ ddlEventProbe: { observedOperationTypes: [...new Set(seen)], markerSeen } }, { depth: null });
+
+        // The marker proves the stream is live and has reached past the DDL, so
+        // the absence of DDL events below is meaningful.
+        expect(markerSeen, 'post-DDL marker event was never delivered — the change stream stalled').toBe(true);
+        // Documents the current limitation. If either fails, DocumentDB has
+        // started delivering DDL events — add drop/rename support and flip these.
+        expect(seen, '`drop` event was delivered — DocumentDB may now support DDL events').not.toContain('drop');
+        expect(seen, '`rename` event was delivered — DocumentDB may now support DDL events').not.toContain('rename');
+      } finally {
+        await cursor.close();
+      }
+    } finally {
+      await cleanup();
+      await client.close();
+    }
+  });
+
+  // Remote DocumentDB clusters can have 10-30s latency spikes for change
+  // stream delivery. Tests that poll for data need headroom.
+  describeWithStorage({ timeout: testTimeout(120_000, { cloudOverride: 180_000 }) }, defineDocumentDBDbModeTests);
+});
+
+function defineDocumentDBDbModeTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
+    return ChangeStreamTestContext.open(factory, {
+      ...options,
+      storageVersion,
+      streamOptions: {
+        ...options?.streamOptions
+      }
+    });
+  };
+
+  test('basic replication in documentDbMode', async () => {
+    await using context = await openContext();
+    const { db } = context;
+    await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data"`);
+
+    await db.createCollection('test_data');
+    const collection = db.collection('test_data');
+
+    await context.replicateSnapshot();
+    context.startStreaming();
+
+    const result = await collection.insertOne({ description: 'test1' });
+    const test_id = result.insertedId;
+    await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
+    await collection.deleteOne({ _id: test_id });
+
+    const data = await context.getBucketData('global[]', undefined, { timeout: DOCUMENTDB_BUCKET_DATA_TIMEOUT });
+
+    expect(data).toMatchObject([
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test1' }),
+      test_utils.putOp('test_data', { id: test_id.toHexString(), description: 'test2' }),
+      test_utils.removeOp('test_data', test_id.toHexString())
+    ]);
+  });
+
+  test('sentinel checkpoint resolution', async () => {
+    await using context = await openContext();
+    const { db } = context;
+    await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data"`);
+
+    await db.createCollection('test_data');
+    const collection = db.collection('test_data');
+
+    await context.replicateSnapshot();
+    context.startStreaming();
+
+    const insertResult = await collection.insertOne({ description: 'sentinel_test' });
+    const insertedId = insertResult.insertedId.toHexString();
+
+    // getCheckpoint() internally calls createCheckpoint, which should return a sentinel
+    // format on DocumentDB. The streaming loop must resolve it by matching the sentinel event.
+    const checkpoint = await context.getCheckpoint();
+    expect(checkpoint).toBeTruthy();
+
+    const data = await context.getBucketData('global[]', undefined, { timeout: DOCUMENTDB_BUCKET_DATA_TIMEOUT });
+    expect(data).toMatchObject([test_utils.putOp('test_data', { id: insertedId, description: 'sentinel_test' })]);
+  });
+
+  // DocumentDB does not support $changeStreamSplitLargeEvent, so large change
+  // events cannot be split into fragments the way the standard MongoDB path
+  // does. This verifies that a large document still replicates end-to-end —
+  // both on insert (the event carries the full document) and on update
+  // (updateLookup fetches the full document into the event) — and that the
+  // large value is persisted through to bucket storage, not just delivered on
+  // the change stream.
+  //
+  // The payload is sized just under PowerSync's MAX_ROW_SIZE (15 MiB). Rows at
+  // or above that limit are dropped by bucket storage regardless of source DB
+  // ("Row too large ... Removing"), so the persisted value cannot approach the
+  // 16 MiB BSON document limit. 14 MiB still forces a large change event
+  // through the DocumentDB stream while keeping the projected row under the limit.
+  // Assertions check the payload length rather than inlining a 14MB string.
+  // Note: DocumentDB delivers large change events very slowly (observed ~18s to fetch a
+  // single ~14 MiB event), so this test allows generous checkpoint timeouts. See
+  // docs/documentdb/documentdb-limitations.md.
+  test(
+    'replicates a large document near the row size limit',
+    // Fetching the large row takes very long in DocumentDB cloud.
+    { timeout: DOCUMENTDB_LARGE_DOCUMENT_TIMEOUT },
+    async () => {
+      await using context = await openContext({
+        streamOptions: {
+          // The test context normally uses 200ms so local MongoDB streams abort
+          // quickly, but DocumentDB needs a realistic getMore maxTimeMS for
+          // intentionally large change events.
+          maxAwaitTimeMS: DOCUMENTDB_LARGE_DOCUMENT_MAX_AWAIT_TIME_MS
+        }
+      });
+      const { db } = context;
+      await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, marker, payload FROM "test_data"`);
+
+      await db.createCollection('test_data');
+      const collection = db.collection('test_data');
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      // 14 MiB payload — large enough to exercise the large-event path, but the
+      // projected row (payload + small envelope) stays under MAX_ROW_SIZE so it
+      // is not dropped by bucket storage.
+      const largePayload = 'x'.repeat(14 * 1024 * 1024);
+
+      const insertResult = await collection.insertOne({ marker: 'big_insert', payload: largePayload });
+      const id = insertResult.insertedId.toHexString();
+
+      // Large events are slow to fetch on DocumentDB, so allow well beyond the 15s default.
+      const afterInsert = await context.getBucketData('global[]', undefined, {
+        timeout: DOCUMENTDB_LARGE_BUCKET_DATA_TIMEOUT
+      });
+      expect(afterInsert.length).toEqual(1);
+      const insertData = JSON.parse(afterInsert[0].data as string);
+      expect(insertData).toMatchObject({ id, marker: 'big_insert' });
+      expect(insertData.payload.length).toEqual(largePayload.length);
+
+      // Update a small field; the payload stays large, so the updateLookup
+      // fullDocument on the change event is still ~15MB.
+      await collection.updateOne({ _id: insertResult.insertedId }, { $set: { marker: 'big_update' } });
+
+      // Fetching the large row takes very long in DocumentDB cloud.
+      const afterUpdate = await context.getBucketData('global[]', undefined, {
+        timeout: DOCUMENTDB_LARGE_BUCKET_DATA_TIMEOUT
+      });
+      expect(afterUpdate.length).toEqual(2);
+      const updateData = JSON.parse(afterUpdate[1].data as string);
+      expect(updateData).toMatchObject({ id, marker: 'big_update' });
+      expect(updateData.payload.length).toEqual(largePayload.length);
+    }
+  );
+
+  test('keepalive in documentDbMode', async () => {
+    await using context = await openContext({
+      streamOptions: {
+        keepaliveIntervalMs: 2000
+      }
+    });
+    const { db } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await db.createCollection('test_data');
+
+    await context.replicateSnapshot();
+    context.startStreaming();
+
+    // Wait for the initial checkpoint to be processed
+    await context.getCheckpoint({ timeout: DOCUMENTDB_BUCKET_DATA_TIMEOUT });
+
+    // Wait past the keepalive interval so the idle keepalive path fires.
+    // On DocumentDB, this must NOT crash from parseResumeTokenTimestamp
+    // (DocumentDB resume tokens are base64, not hex).
+    await setTimeout(3000);
+
+    // Insert data after the keepalive interval to verify the stream is still alive
+    const collection = db.collection('test_data');
+    await collection.insertOne({ description: 'after_keepalive' });
+
+    const data = await context.getBucketData('global[]', undefined, {
+      timeout: DOCUMENTDB_BUCKET_DATA_TIMEOUT
+    });
+    expect(data.length).toBeGreaterThanOrEqual(1);
+    const lastOp = data[data.length - 1];
+    expect(JSON.parse(lastOp.data as string)).toMatchObject({ description: 'after_keepalive' });
+  });
+
+  // Skipped until Azure DocumentDB ships the server-side getMore maxAwaitTimeMS fix.
+  test.skip('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
+    const maxAwaitTimeMS = 2_000;
+
+    await using context = await openContext({
+      streamOptions: {
+        maxAwaitTimeMS
+      }
+    });
+
+    // DocumentDB uses a cluster-level change stream through client.db('admin'), so
+    // spying on only context.db.command would miss the getMore calls. Spying on
+    // the Db prototype captures command calls from both the test DB and admin DB
+    // while still delegating to the real MongoDB driver implementation.
+    const dbPrototype = Object.getPrototypeOf(context.db);
+    const originalCommand = dbPrototype.command;
+    const getMoreTimings: {
+      collection: unknown;
+      maxTimeMS: unknown;
+      durationMS: number;
+      nextBatchLength: number | undefined;
+    }[] = [];
+    const commandSpy = vi.spyOn(dbPrototype, 'command').mockImplementation(async function (
+      this: unknown,
+      command: any,
+      options?: any
+    ) {
+      if (command?.getMore == null) {
+        return originalCommand.call(this, command, options);
+      }
+
+      // Measure the actual round-trip duration of the driver's getMore command.
+      // This verifies the server waits for maxTimeMS when the change stream is
+      // idle, rather than returning empty batches immediately.
+      const start = performance.now();
+      let result: any;
+      try {
+        result = await originalCommand.call(this, command, options);
+        return result;
+      } finally {
+        const cursor = Buffer.isBuffer(result?.cursor)
+          ? mongo.BSON.deserialize(result.cursor, {
+              useBigInt64: true,
+              fieldsAsRaw: { nextBatch: true },
+              validation: { utf8: false }
+            })
+          : result?.cursor;
+
+        getMoreTimings.push({
+          collection: command.collection,
+          maxTimeMS: command.maxTimeMS,
+          durationMS: Math.round(performance.now() - start),
+          nextBatchLength: cursor?.nextBatch?.length
+        });
+      }
+    });
+
+    try {
+      const { db } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await db.createCollection('test_data');
+      const collection = db.collection('test_data');
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const result = await collection.insertOne({ description: 'maxAwaitTimeMS_test' });
+      await context.getBucketData('global[]');
+
+      // Once the stream has caught up, the latest getMore call should eventually
+      // be an idle poll. That idle poll should wait for maxTimeMS instead of
+      // returning immediately, and the response should contain an empty batch.
+      await vi.waitFor(
+        () => {
+          const lastGetMore = getMoreTimings.at(-1);
+          expect(lastGetMore?.durationMS).toBeGreaterThanOrEqual(maxAwaitTimeMS);
+          expect(lastGetMore?.nextBatchLength).toEqual(0);
+        },
+        {
+          timeout: maxAwaitTimeMS + 2_000,
+          interval: 100
+        }
+      );
+
+      console.dir({ getMoreMaxAwaitTimeMSTimings: getMoreTimings }, { depth: null });
+
+      expect(result.insertedId).toBeTruthy();
+      expect(getMoreTimings.length).toBeGreaterThan(0);
+    } finally {
+      commandSpy.mockRestore();
+    }
+  });
+
+  test('write checkpoint flow in documentDbMode', async () => {
+    await using context = await openContext();
+    const { db } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await db.createCollection('test_data');
+    const collection = db.collection('test_data');
+
+    await context.replicateSnapshot();
+    await context.markSnapshotConsistent();
+
+    await using api = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS
+    });
+
+    context.startStreaming();
+
+    // Wait until stream is active
+    await context.getCheckpoint();
+
+    // Insert data so the stream has something to process
+    await collection.insertOne({ description: 'write_cp_test' });
+
+    const writeCheckpointBatcher = new WriteCheckpointBatcher(
+      () => api,
+      () => context.factory
+    );
+
+    // Exercise the write checkpoint flow. On DocumentDB, createReplicationHead
+    // advances the sentinel counter with a standalone (null stream_id) bump and
+    // uses the resulting sentinel-based LSN as the source-side head. The sentinel
+    // write also nudges replication forward on an idle stream.
+    const result = await createWriteCheckpoint({
+      userId: 'test_user',
+      clientId: 'test_client',
+      batcher: writeCheckpointBatcher
+    });
+
+    // The write checkpoint should resolve with a valid result
+    expect(result).toBeTruthy();
+    expect(result.writeCheckpoint).toBeTruthy();
+    expect(result.replicationHead).toBeTruthy();
+  });
+
+  test('data events not dropped after restart (lte guard)', async () => {
+    // Verifies that the .lte() dedup guard in streamChangesInternal does NOT
+    // drop data events on DocumentDB after restart. On DocumentDB, wallTime has
+    // second precision (increment 0). Without the isDocumentDb guard, events
+    // within the same wall-clock second as the last checkpoint would be silently
+    // dropped — causing data loss.
+    //
+    // This test avoids getClientCheckpoint (which has its own timing issues)
+    // and instead polls the storage directly until the data appears.
+
+    // Phase 1: initial sync + streaming + checkpoint
+    await using context = await openContext();
+    const { db } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await db.createCollection('test_data');
+    const collection = db.collection('test_data');
+
+    await context.replicateSnapshot();
+    context.startStreaming();
+
+    await collection.insertOne({ description: 'phase1_data' });
+    await context.getCheckpoint();
+
+    // Stop streaming
+    context.abort();
+    await context.dispose();
+
+    // Phase 2: restart and insert immediately (same second as last checkpoint)
+    await using context2 = await openContext({ doNotClear: true });
+    const db2 = context2.db;
+
+    await context2.loadActiveSyncRules();
+
+    context2.startStreaming();
+
+    // Wait for the stream to be fully initialized — the streaming loop must
+    // process its initial checkpoint before we insert test data. Without this,
+    // the insert can happen before the ChangeStream's lazy aggregate command
+    // is sent, causing the event to be missed entirely (not a .lte() issue).
+    await context2.getCheckpoint({ timeout: DOCUMENTDB_RESTART_TIMEOUT });
+
+    // Insert — if .lte() drops same-second events, this data will never appear.
+    const collection2 = db2.collection('test_data');
+    const result2 = await collection2.insertOne({ description: 'post_restart_data' });
+    const id2 = result2.insertedId;
+
+    // Poll for the data by repeatedly calling getBucketData with a longer timeout.
+    // We bypass the flaky getClientCheckpoint timing by polling until the data appears
+    // or the timeout expires. If the .lte() guard drops same-second events, the data
+    // will never appear — deterministic failure.
+    const deadline = Date.now() + DOCUMENTDB_RESTART_TIMEOUT;
+    let found = false;
+    while (Date.now() < deadline) {
+      try {
+        const data = await context2.getBucketData('global[]', undefined, { timeout: 5_000 });
+        const match = data.find((op) => op.object_id === id2.toHexString() && op.op === 'PUT');
+        if (match) {
+          const parsed = JSON.parse(match.data as string);
+          expect(parsed).toMatchObject({ description: 'post_restart_data' });
+          found = true;
+          break;
+        }
+      } catch {
+        // getCheckpoint may timeout on first attempts — retry
+      }
+      await setTimeout(200);
+    }
+
+    expect(
+      found,
+      'Data event after restart was dropped — .lte() guard may be incorrectly filtering same-second events'
+    ).toBe(true);
+  });
+
+  test('resume after restart in documentDbMode', async () => {
+    // Phase 1: replicate some data, then stop
+    await using context = await openContext();
+    const { db } = context;
+    await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT _id as id, description FROM "test_data"`);
+
+    await db.createCollection('test_data');
+    const collection = db.collection('test_data');
+
+    await context.replicateSnapshot();
+    context.startStreaming();
+
+    const result1 = await collection.insertOne({ description: 'before_restart' });
+    const id1 = result1.insertedId;
+
+    // Wait for the data to be replicated and checkpoint to advance
+    await context.getCheckpoint({ timeout: DOCUMENTDB_BUCKET_DATA_TIMEOUT });
+
+    const dataBefore = await context.getBucketDataAtLatestCheckpoint('global[]');
+    expect(dataBefore).toMatchObject([
+      test_utils.putOp('test_data', { id: id1.toHexString(), description: 'before_restart' })
+    ]);
+
+    // Stop streaming (simulates a restart)
+    context.abort();
+    await context.dispose();
+
+    // Phase 2: reopen without clearing and resume
+    await using context2 = await openContext({ doNotClear: true });
+    const db2 = context2.db;
+
+    await context2.loadActiveSyncRules();
+
+    context2.startStreaming();
+
+    // Wait for the stream to fully initialize and process the initial checkpoint
+    // before inserting new data.
+    await context2.getCheckpoint({ timeout: DOCUMENTDB_RESTART_TIMEOUT });
+
+    const collection2 = db2.collection('test_data');
+    const result2 = await collection2.insertOne({ description: 'after_restart' });
+    const id2 = result2.insertedId;
+
+    // On DocumentDB, wall-clock LSNs have second precision and stored LSNs may
+    // include resume-token suffixes. Avoid creating a fresh sentinel checkpoint
+    // on every poll; read at the latest persisted checkpoint instead.
+    const deadline = Date.now() + DOCUMENTDB_RESTART_TIMEOUT;
+    let found = false;
+    while (Date.now() < deadline) {
+      const dataAfter = await context2.getBucketDataAtLatestCheckpoint('global[]');
+      const afterRestartOps = dataAfter.filter((op) => op.object_id === id2.toHexString() && op.op === 'PUT');
+      if (afterRestartOps.length >= 1) {
+        expect(JSON.parse(afterRestartOps[0].data as string)).toMatchObject({ description: 'after_restart' });
+        found = true;
+        break;
+      }
+      await setTimeout(200);
+    }
+    expect(found, 'Data event after restart was not replicated within timeout').toBe(true);
+  });
+}