@powersync/service-module-mongodb 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/CHANGELOG.md +153 -5
  2. package/dist/api/MongoRouteAPIAdapter.d.ts +4 -2
  3. package/dist/api/MongoRouteAPIAdapter.js +23 -32
  4. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  5. package/dist/common/SentinelLSN.d.ts +37 -0
  6. package/dist/common/SentinelLSN.js +59 -0
  7. package/dist/common/SentinelLSN.js.map +1 -0
  8. package/dist/module/MongoModule.js +2 -1
  9. package/dist/module/MongoModule.js.map +1 -1
  10. package/dist/replication/ChangeStream.d.ts +24 -0
  11. package/dist/replication/ChangeStream.js +203 -67
  12. package/dist/replication/ChangeStream.js.map +1 -1
  13. package/dist/replication/ChangeStreamReplicationJob.js +3 -2
  14. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  15. package/dist/replication/ChangeStreamReplicator.js +1 -1
  16. package/dist/replication/ChangeStreamReplicator.js.map +1 -1
  17. package/dist/replication/MongoRelation.d.ts +36 -1
  18. package/dist/replication/MongoRelation.js +133 -18
  19. package/dist/replication/MongoRelation.js.map +1 -1
  20. package/dist/replication/MongoSnapshotter.d.ts +9 -1
  21. package/dist/replication/MongoSnapshotter.js +129 -67
  22. package/dist/replication/MongoSnapshotter.js.map +1 -1
  23. package/dist/replication/RawChangeStream.d.ts +4 -1
  24. package/dist/replication/RawChangeStream.js +6 -10
  25. package/dist/replication/RawChangeStream.js.map +1 -1
  26. package/dist/replication/checkpoints/CheckpointImplementation.d.ts +135 -0
  27. package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
  28. package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
  29. package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +58 -0
  30. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +224 -0
  31. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
  32. package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +27 -0
  33. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +116 -0
  34. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
  35. package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
  36. package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
  37. package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
  38. package/dist/replication/replication-utils.d.ts +6 -0
  39. package/dist/replication/replication-utils.js +19 -2
  40. package/dist/replication/replication-utils.js.map +1 -1
  41. package/dist/types/types.d.ts +7 -2
  42. package/dist/types/types.js +20 -2
  43. package/dist/types/types.js.map +1 -1
  44. package/package.json +11 -21
  45. package/src/api/MongoRouteAPIAdapter.ts +28 -39
  46. package/src/common/SentinelLSN.ts +78 -0
  47. package/src/module/MongoModule.ts +2 -1
  48. package/src/replication/ChangeStream.ts +223 -78
  49. package/src/replication/ChangeStreamReplicationJob.ts +3 -2
  50. package/src/replication/ChangeStreamReplicator.ts +1 -1
  51. package/src/replication/MongoRelation.ts +153 -21
  52. package/src/replication/MongoSnapshotter.ts +148 -73
  53. package/src/replication/RawChangeStream.ts +27 -32
  54. package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
  55. package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +269 -0
  56. package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +145 -0
  57. package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
  58. package/src/replication/replication-utils.ts +23 -2
  59. package/src/types/types.ts +27 -2
  60. package/test/DOCUMENTDB_TESTING.md +115 -0
  61. package/test/src/DatabaseType.ts +25 -0
  62. package/test/src/change_stream.test.ts +180 -71
  63. package/test/src/change_stream_utils.ts +117 -29
  64. package/test/src/checkpoint_retry.test.ts +134 -0
  65. package/test/src/config.test.ts +34 -0
  66. package/test/src/documentdb_helpers.test.ts +124 -0
  67. package/test/src/documentdb_mode.test.ts +1039 -0
  68. package/test/src/mongo_test.test.ts +15 -5
  69. package/test/src/raw_change_stream.test.ts +147 -126
  70. package/test/src/resume.test.ts +2 -2
  71. package/test/src/resume_token.test.ts +30 -0
  72. package/test/src/slow_tests.test.ts +4 -1
  73. package/test/src/test-timeouts.ts +23 -0
  74. package/test/src/util.ts +14 -32
  75. package/tsconfig.json +1 -4
  76. package/tsconfig.scripts.json +1 -2
  77. package/tsconfig.tsbuildinfo +1 -1
@@ -13,9 +13,10 @@ import {
13
13
  TimeValuePrecision
14
14
  } from '@powersync/service-sync-rules';
15
15
 
16
- import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
16
+ import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
17
+ import { ObjectId } from 'bson';
17
18
  import { MongoLSN } from '../common/MongoLSN.js';
18
- import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
19
+ import { SentinelLSN } from '../common/SentinelLSN.js';
19
20
 
20
21
  export function getMongoRelation(
21
22
  source: mongo.ChangeStreamNameSpace,
@@ -174,32 +175,163 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
174
175
  */
175
176
  export const STANDALONE_CHECKPOINT_ID = '_standalone_checkpoint';
176
177
 
177
- export async function createCheckpoint(
178
+ /**
179
+ * Id for checkpoint records managed by the {@link SentinelCheckpointImplementation} implementation.
180
+ */
181
+ export const SENTINEL_CHECKPOINT_ID = '_sentinel_checkpoint';
182
+
183
+ /**
184
+ * Create a checkpoint by upserting a document in _powersync_checkpoints, and
185
+ * return a comparable LSN string derived from the write's operationTime.
186
+ *
187
+ * Standard MongoDB only: this requires session.operationTime, which DocumentDB
188
+ * does not provide (it throws PSYNC_S1004 when it is missing). The DocumentDB /
189
+ * sentinel path builds LSNs via {@link createSentinelCheckpointLsn} instead.
190
+ */
191
+ export async function createCheckpoint(db: mongo.Db, id: mongo.ObjectId | string): Promise<string> {
192
+ const TRIES = 2;
193
+ for (let i = 0; i < TRIES; i++) {
194
+ try {
195
+ return await createCheckpointInner(db, id);
196
+ } catch (e) {
197
+ if (i < TRIES - 1) {
198
+ logger.warn(`Failed to create checkpoint on attempt ${i + 1}`, e);
199
+ } else {
200
+ throw e;
201
+ }
202
+ }
203
+ }
204
+ throw new ServiceAssertionError(`Unreachable code`);
205
+ }
206
+
207
+ async function createCheckpointInner(db: mongo.Db, id: mongo.ObjectId | string): Promise<string> {
208
+ // We use an unique id per process, and clear documents on startup.
209
+ // This is so that we can filter events for our own process only, and ignore
210
+ // events from other processes.
211
+
212
+ // We use a command instead of a regular update to avoid auto retries on writes.
213
+ // An auto retry on the write can trigger a weird edge case where the change stream event
214
+ // has the clusterTime of the first write, while the returned operation time is for the second no-op write.
215
+ // Instead, we do manual retries, which does not have the same write de-duplication logic.
216
+ // A sentinal-based approach would be better here, but that is a much bigger change.
217
+
218
+ const update: mongo.Document = { $inc: { i: 1 } };
219
+
220
+ const response = await db.command({
221
+ findAndModify: '_powersync_checkpoints',
222
+ query: {
223
+ _id: id as any
224
+ },
225
+ new: true,
226
+ upsert: true,
227
+ update
228
+ });
229
+
230
+ const time = response.operationTime as mongo.Timestamp | undefined;
231
+ if (time == null) {
232
+ throw new ServiceError(ErrorCode.PSYNC_S1004, `clusterTime not available for checkpoint`);
233
+ }
234
+ return new MongoLSN({ timestamp: time }).comparable;
235
+ }
236
+
237
+ /**
238
+ * Create a DocumentDB comparable LSN by advancing the shared sentinel checkpoint
239
+ * document ({@link SENTINEL_CHECKPOINT_ID}). The returned LSN encodes the
240
+ * checkpoint counter in the same 16-hex shape as a MongoDB timestamp LSN (see
241
+ * {@link SentinelLSN}), so the two formats are directly comparable. The counter is
242
+ * seeded in the epoch-seconds range (see below), so a sentinel LSN always sorts
243
+ * above any real-timestamp LSN issued in the past.
244
+ *
245
+ * This counter is intentionally global to the source database. It is used for
246
+ * storage/client checkpoint comparisons and write checkpoint heads, so it must
247
+ * not reset when a new ChangeStream instance or new sync rules start.
248
+ *
249
+ * The document is a single shared record whose `stream_id` field alternates:
250
+ * batch/keepalive bumps stamp it with the calling stream's id (so a stream can
251
+ * recognise its own private barriers), while standalone bumps (write checkpoint
252
+ * heads, snapshot markers) clear it to null. `i` advances globally regardless.
253
+ *
254
+ * @param changeStreamId
255
+ * When provided, the bump is attributed to this stream (a private barrier).
256
+ * When omitted, it is a standalone bump and stream_id is cleared to null.
257
+ */
258
+ export async function createSentinelCheckpointLsn(
178
259
  client: mongo.MongoClient,
179
260
  db: mongo.Db,
180
- id: mongo.ObjectId | string
261
+ changeStreamId?: ObjectId
181
262
  ): Promise<string> {
182
263
  const session = client.startSession();
183
264
  try {
184
- // We use an unique id per process, and clear documents on startup.
185
- // This is so that we can filter events for our own process only, and ignore
186
- // events from other processes.
187
- await db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
188
- {
189
- _id: id as any
190
- },
191
- {
192
- $inc: { i: 1 }
193
- },
194
- {
195
- upsert: true,
196
- returnDocument: 'after',
197
- session
265
+ const collection = db.collection('_powersync_checkpoints');
266
+
267
+ for (let attempt = 0; attempt < 3; attempt++) {
268
+ // Common path: increment the existing counter.
269
+ const result = await collection.findOneAndUpdate(
270
+ {
271
+ _id: SENTINEL_CHECKPOINT_ID as any,
272
+ i: { $exists: true }
273
+ },
274
+ {
275
+ $inc: { i: 1 },
276
+ // Standalone bumps (no changeStreamId) must explicitly clear the
277
+ // stream_id left by a previous batch bump — this is a single shared
278
+ // document whose stream_id alternates. Write null rather than relying
279
+ // on `undefined` being serialized (which depends on the driver's
280
+ // ignoreUndefined option, and collapses to an empty $set if enabled).
281
+ $set: {
282
+ stream_id: changeStreamId ?? null
283
+ }
284
+ },
285
+ {
286
+ returnDocument: 'after',
287
+ session
288
+ }
289
+ );
290
+
291
+ if (result != null) {
292
+ // `i` is a bigint: the client is configured with useBigInt64, and the
293
+ // seed exceeds 2^53 so it could not be safely represented as a number.
294
+ return new SentinelLSN({ sentinel: result.i }).comparable;
198
295
  }
296
+
297
+ // The counter document does not exist: first run, or a consumer deleted
298
+ // it in their source database. Seed the counter in the epoch-SECONDS range
299
+ // (seconds in the high 32 bits, mirroring a MongoDB timestamp) rather than
300
+ // starting at 1. Two properties follow:
301
+ //
302
+ // - A sentinel LSN sorts above any real-timestamp LSN issued in the past,
303
+ // because its high 32 bits are the current epoch seconds.
304
+ // - A re-created counter jumps forward instead of backward across deletion:
305
+ // the seed advances by 2^32 each wall-clock second, while checkpoints add
306
+ // 1 each, so any later re-seed exceeds the previously issued coordinate
307
+ // (keeping the LSN domain monotonic; otherwise new write checkpoint heads
308
+ // could resolve against old, higher committed LSNs).
309
+ //
310
+ // $setOnInsert cannot be combined with $inc on the same field, so this
311
+ // is a separate upsert; the loop then retries the increment. The
312
+ // $setOnInsert is a no-op if another process created the document
313
+ // concurrently.
314
+ await collection.updateOne(
315
+ {
316
+ _id: SENTINEL_CHECKPOINT_ID as any
317
+ },
318
+ {
319
+ $setOnInsert: {
320
+ i: BigInt(Math.floor(Date.now() / 1000)) << 32n,
321
+ stream_id: changeStreamId ?? null
322
+ }
323
+ },
324
+ {
325
+ upsert: true,
326
+ session
327
+ }
328
+ );
329
+ }
330
+
331
+ throw new ServiceError(
332
+ ErrorCode.PSYNC_S1301,
333
+ `Failed to increment the sentinel checkpoint counter - the checkpoint document may be getting deleted concurrently.`
199
334
  );
200
- const time = session.operationTime!;
201
- // TODO: Use the above when we support custom write checkpoints
202
- return new MongoLSN({ timestamp: time }).comparable;
203
335
  } finally {
204
336
  await session.endSession();
205
337
  }
@@ -1,7 +1,6 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { container, ErrorCode, Logger, ReplicationAbortedError, ServiceError } from '@powersync/lib-services-framework';
3
3
  import {
4
- InternalOpId,
5
4
  MetricsEngine,
6
5
  PerformanceTracer,
7
6
  SaveOperationTag,
@@ -15,11 +14,13 @@ import { performance } from 'node:perf_hooks';
15
14
  import { MongoLSN } from '../common/MongoLSN.js';
16
15
  import { PostImagesOption } from '../types/types.js';
17
16
  import { escapeRegExp } from '../utils.js';
17
+ import { CheckpointImplementation } from './checkpoints/CheckpointImplementation.js';
18
+ import { createCheckpointImplementation } from './checkpoints/create-checkpoint-implementation.js';
18
19
  import { MongoManager } from './MongoManager.js';
19
- import { createCheckpoint, getMongoRelation, STANDALONE_CHECKPOINT_ID } from './MongoRelation.js';
20
+ import { getMongoRelation } from './MongoRelation.js';
20
21
  import { ChunkedSnapshotQuery } from './MongoSnapshotQuery.js';
21
22
  import { ChangeStreamBatch, parseChangeDocument, rawChangeStream } from './RawChangeStream.js';
22
- import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
23
+ import { CHECKPOINTS_COLLECTION, detectDocumentDb } from './replication-utils.js';
23
24
  import { DirectSourceRowConverter, SourceRowConverter } from './SourceRowConverter.js';
24
25
 
25
26
  export interface MongoSnapshotterOptions {
@@ -71,9 +72,11 @@ export class MongoSnapshotter {
71
72
  private readonly queue = new Set<SnapshotQueueItem>();
72
73
  private initialSnapshotDone = Promise.withResolvers<void>();
73
74
  private nextItemQueued: PromiseWithResolvers<void> | null = null;
74
- private lastSnapshotOpId: InternalOpId | null = null;
75
75
  private lastTouchedAt = performance.now();
76
76
 
77
+ private isDocumentDb = false;
78
+ private _checkpointImplementation: CheckpointImplementation | null = null;
79
+
77
80
  constructor(options: MongoSnapshotterOptions) {
78
81
  this.storage = options.storage;
79
82
  this.metrics = options.metrics;
@@ -96,6 +99,9 @@ export class MongoSnapshotter {
96
99
  this.abortSignal.addEventListener('abort', () => {
97
100
  this.nextItemQueued?.resolve();
98
101
  });
102
+
103
+ // Errors here should not result in uncaught rejection - calling waitForInitialSnapshot is optional.
104
+ void this.initialSnapshotDone.promise.catch(() => {});
99
105
  }
100
106
 
101
107
  private get usePostImages() {
@@ -110,23 +116,71 @@ export class MongoSnapshotter {
110
116
  return this.storage.storageConfig.softDeleteCurrentData;
111
117
  }
112
118
 
119
+ /** The active checkpoint strategy. Only valid after ensureDetected(). */
120
+ private get checkpointImplementation(): CheckpointImplementation {
121
+ if (this._checkpointImplementation == null) {
122
+ throw new ServiceError(
123
+ ErrorCode.PSYNC_S1301,
124
+ 'Checkpoint implementation not initialized - call ensureDetected() first'
125
+ );
126
+ }
127
+ return this._checkpointImplementation;
128
+ }
129
+
130
+ /**
131
+ * Detect DocumentDB and select the checkpoint implementation. Idempotent.
132
+ * Also validates server topology (sharded/standalone) for standard MongoDB.
133
+ */
134
+ private async ensureDetected(): Promise<void> {
135
+ if (this._checkpointImplementation != null) {
136
+ return;
137
+ }
138
+ this.isDocumentDb = await detectDocumentDb(this.defaultDb);
139
+ if (!this.isDocumentDb) {
140
+ const hello = await this.defaultDb.command({ hello: 1 });
141
+ if (hello.msg == 'isdbgrid') {
142
+ throw new ServiceError(
143
+ ErrorCode.PSYNC_S1341,
144
+ 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
145
+ );
146
+ } else if (hello.setName == null) {
147
+ throw new ServiceError(
148
+ ErrorCode.PSYNC_S1342,
149
+ 'Standalone MongoDB instances are not supported - use a replicaset.'
150
+ );
151
+ }
152
+ }
153
+ this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
154
+ client: this.client,
155
+ db: this.defaultDb,
156
+ checkpointStreamId: this.checkpointStreamId,
157
+ logger: this.logger
158
+ });
159
+ }
160
+
113
161
  async checkSlot(): Promise<InitResult> {
114
162
  const status = await this.storage.getStatus();
115
- if (status.snapshot_done && status.checkpoint_lsn) {
163
+ if (status.snapshotDone) {
116
164
  this.logger.info(`Initial replication already done`);
117
165
  return { needsInitialSync: false, snapshotLsn: null };
118
166
  }
119
167
 
120
- return { needsInitialSync: true, snapshotLsn: status.snapshot_lsn };
168
+ return { needsInitialSync: true, snapshotLsn: status.resumeLsn };
121
169
  }
122
170
 
123
171
  async setupCheckpointsCollection() {
172
+ await this.ensureDetected();
124
173
  const collection = await this.getCollectionInfo(this.defaultDb.databaseName, CHECKPOINTS_COLLECTION);
125
174
  if (collection == null) {
126
175
  await this.defaultDb.createCollection(CHECKPOINTS_COLLECTION, {
127
- changeStreamPreAndPostImages: { enabled: true }
176
+ // DocumentDB does not support changeStreamPreAndPostImages.
177
+ ...(this.isDocumentDb ? {} : { changeStreamPreAndPostImages: { enabled: true } })
128
178
  });
129
- } else if (this.usePostImages && collection.options?.changeStreamPreAndPostImages?.enabled != true) {
179
+ } else if (
180
+ !this.isDocumentDb &&
181
+ this.usePostImages &&
182
+ collection.options?.changeStreamPreAndPostImages?.enabled != true
183
+ ) {
130
184
  // Drop + create requires less permissions than collMod,
131
185
  // and we don't care about the data in this collection.
132
186
  await this.defaultDb.dropCollection(CHECKPOINTS_COLLECTION);
@@ -137,17 +191,28 @@ export class MongoSnapshotter {
137
191
  // Clear the collection on startup, to keep it clean
138
192
  // We never query this collection directly, and don't want to keep the data around.
139
193
  // We only use this to get data into the oplog/changestream.
140
- await this.defaultDb.collection(CHECKPOINTS_COLLECTION).deleteMany({});
194
+ //
195
+ // The implementation supplies the filter: the sentinel implementation must
196
+ // preserve the sentinel checkpoint document, since its counter is the
197
+ // globally-ordered component of every committed DocumentDB LSN.
198
+ await this.defaultDb
199
+ .collection(CHECKPOINTS_COLLECTION)
200
+ .deleteMany(this.checkpointImplementation.checkpointClearFilter);
141
201
  }
142
202
  }
143
203
 
144
204
  async queueSnapshotTables(snapshotLsn: string | null) {
145
205
  await this.client.connect();
206
+ // Ensure isDocumentDb is set before any getSourceNamespaceFilters() call below
207
+ // (notably the validateSnapshotLsn resume path, which does not go through
208
+ // getSnapshotLsn). Idempotent.
209
+ await this.ensureDetected();
146
210
  await using writer = await this.storage.createWriter({
147
211
  zeroLSN: MongoLSN.ZERO.comparable,
148
212
  defaultSchema: this.defaultDb.databaseName,
149
213
  storeCurrentData: false,
150
214
  skipExistingRows: true,
215
+ signal: this.abortSignal,
151
216
  tracer: new PerformanceTracer('MongoDB initial snapshot setup')
152
217
  });
153
218
  if (snapshotLsn == null) {
@@ -258,7 +323,7 @@ export class MongoSnapshotter {
258
323
  for (const table of tables) {
259
324
  await this.snapshotTable(batch, table);
260
325
  }
261
- const noCheckpointBefore = await createCheckpoint(this.client, this.defaultDb, STANDALONE_CHECKPOINT_ID);
326
+ const noCheckpointBefore = await this.checkpointImplementation.createStandaloneCheckpoint();
262
327
 
263
328
  await batch.markTableSnapshotDone(tables, noCheckpointBefore);
264
329
  }
@@ -277,20 +342,16 @@ export class MongoSnapshotter {
277
342
  }
278
343
 
279
344
  const status = await this.storage.getStatus();
280
- if (status.snapshot_done) {
345
+ if (status.snapshotDone) {
281
346
  return;
282
347
  }
283
348
 
284
- const lastOp = this.lastSnapshotOpId ?? status.keepalive_op;
285
- if (lastOp != null) {
286
- // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream.
287
- // Keeping snapshot_done false until this completes makes this resumable after interruption.
288
- await this.storage.populatePersistentChecksumCache({
289
- // No checkpoint yet, but we do have the opId.
290
- maxOpId: lastOp,
291
- signal: this.abortSignal
292
- });
293
- }
349
+ // Compact storage _after_ initial replication, but _before_ we switch to this replication stream.
350
+ // Keeping snapshot_done false until this completes makes this resumable after interruption.
351
+ // No checkpoint exists yet - storage defaults to its highest persisted op id.
352
+ await this.storage.compactInitialReplication({
353
+ signal: this.abortSignal
354
+ });
294
355
 
295
356
  if (this.queue.size != 0) {
296
357
  return;
@@ -301,12 +362,13 @@ export class MongoSnapshotter {
301
362
  zeroLSN: MongoLSN.ZERO.comparable,
302
363
  defaultSchema: this.defaultDb.databaseName,
303
364
  storeCurrentData: false,
304
- skipExistingRows: true
365
+ skipExistingRows: true,
366
+ signal: this.abortSignal
305
367
  });
306
368
 
307
369
  // The checkpoint here is a marker - we need to replicate up to at least this
308
370
  // point before the data can be considered consistent.
309
- const checkpoint = await createCheckpoint(this.client, this.defaultDb, STANDALONE_CHECKPOINT_ID);
371
+ const checkpoint = await this.checkpointImplementation.createStandaloneCheckpoint();
310
372
  if (this.queue.size != 0) {
311
373
  return;
312
374
  }
@@ -318,7 +380,7 @@ export class MongoSnapshotter {
318
380
  // KLUDGE: We need to create an extra checkpoint _after_ marking the snapshot done, to fix
319
381
  // issues with order of processing commits(). This is picked up by tests on postgres storage,
320
382
  // the issue may be specific to that storage engine.
321
- await createCheckpoint(this.client, this.defaultDb, STANDALONE_CHECKPOINT_ID);
383
+ await this.checkpointImplementation.createStandaloneCheckpoint();
322
384
  }
323
385
 
324
386
  private async replicateTable(tableRequest: SourceTable) {
@@ -331,6 +393,7 @@ export class MongoSnapshotter {
331
393
  storeCurrentData: false,
332
394
  skipExistingRows: true,
333
395
  hooks: this.storageHooks,
396
+ signal: this.abortSignal,
334
397
  tracer: new PerformanceTracer('MongoDB snapshot table')
335
398
  });
336
399
  // Get fresh table info, in case it was updated while queuing.
@@ -342,16 +405,13 @@ export class MongoSnapshotter {
342
405
  }
343
406
 
344
407
  await this.snapshotTable(writer, table);
345
- const noCheckpointBefore = await createCheckpoint(this.client, this.defaultDb, STANDALONE_CHECKPOINT_ID);
408
+ const noCheckpointBefore = await this.checkpointImplementation.createStandaloneCheckpoint();
346
409
  await writer.markTableSnapshotDone([table], noCheckpointBefore);
347
410
 
348
- // This commit ensures we set keepalive_op.
411
+ // This commit durably records the persisted ops, so a later checkpoint covers them.
349
412
  const resumeLsn = writer.resumeFromLsn ?? MongoLSN.ZERO.comparable;
350
413
  await writer.commit(resumeLsn);
351
414
 
352
- if (writer.last_flushed_op != null) {
353
- this.lastSnapshotOpId = writer.last_flushed_op;
354
- }
355
415
  this.logger.info(`Flushed snapshot at ${writer.last_flushed_op}`);
356
416
  }
357
417
 
@@ -364,6 +424,13 @@ export class MongoSnapshotter {
364
424
  return [];
365
425
  }
366
426
 
427
+ if (tablePattern.isSchemaWildcard) {
428
+ throw new ServiceError(
429
+ ErrorCode.PSYNC_R2201,
430
+ 'Schema wildcards ("%") in table patterns are not supported for MongoDB connections.'
431
+ );
432
+ }
433
+
367
434
  const nameFilter = tablePattern.isWildcard
368
435
  ? new RegExp('^' + escapeRegExp(tablePattern.tablePrefix))
369
436
  : tablePattern.name;
@@ -445,10 +512,7 @@ export class MongoSnapshotter {
445
512
  }
446
513
 
447
514
  // Important: flush before marking progress
448
- const result = await batch.flush();
449
- if (result?.flushed_op != null) {
450
- this.lastSnapshotOpId = result.flushed_op;
451
- }
515
+ await batch.flush();
452
516
  at += docBatch.length;
453
517
  rowsReplicatedMetric.add(docBatch.length);
454
518
 
@@ -481,10 +545,12 @@ export class MongoSnapshotter {
481
545
  // Ignore the postImages check in this case.
482
546
  }
483
547
 
548
+ // Note: resolveTables uses the batch's own parsed sync config set. Passing
549
+ // this.syncRules here would pair sources from one parse with the batch's mapping
550
+ // from another parse.
484
551
  const result = await batch.resolveTables({
485
552
  connection_id: this.connectionId,
486
- source: descriptor,
487
- syncRules: this.syncRules
553
+ source: descriptor
488
554
  });
489
555
 
490
556
  // Drop conflicting collections.
@@ -526,19 +592,7 @@ export class MongoSnapshotter {
526
592
  }
527
593
 
528
594
  private async getSnapshotLsn(): Promise<string> {
529
- const hello = await this.defaultDb.command({ hello: 1 });
530
- // Basic sanity check
531
- if (hello.msg == 'isdbgrid') {
532
- throw new ServiceError(
533
- ErrorCode.PSYNC_S1341,
534
- 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
535
- );
536
- } else if (hello.setName == null) {
537
- throw new ServiceError(
538
- ErrorCode.PSYNC_S1342,
539
- 'Standalone MongoDB instances are not supported - use a replicaset.'
540
- );
541
- }
595
+ await this.ensureDetected();
542
596
 
543
597
  // Open a change stream just to get a resume token for later use.
544
598
  // We could use clusterTime from the hello command, but that won't tell us if the
@@ -551,14 +605,19 @@ export class MongoSnapshotter {
551
605
  // Instead, we create a new checkpoint document, and wait until we get that document back in the stream.
552
606
  // To avoid potential race conditions with the checkpoint creation, we create a new checkpoint document
553
607
  // periodically until the timeout is reached.
608
+ //
609
+ // For the sentinel implementation (DocumentDB) there is no
610
+ // startAtOperationTime: the stream opens from "now" (lsn null) and the
611
+ // retry loop below re-creates checkpoints until one is observed.
554
612
 
555
613
  const LSN_TIMEOUT_SECONDS = 60;
556
614
  const LSN_CREATE_INTERVAL_SECONDS = 1;
557
615
 
558
- const firstCheckpointLsn = await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
616
+ this.checkpointImplementation.seedPosition(null);
617
+ const startStreamFromLsn = await this.checkpointImplementation.createFirstBarrier();
559
618
  const filters = this.getSourceNamespaceFilters();
560
619
  const iter = this.rawChangeStreamBatches({
561
- lsn: firstCheckpointLsn,
620
+ lsn: startStreamFromLsn,
562
621
  maxAwaitTimeMS: 0,
563
622
  signal: this.abortSignal,
564
623
  filters
@@ -573,7 +632,7 @@ export class MongoSnapshotter {
573
632
  break;
574
633
  }
575
634
  if (performance.now() - lastCheckpointCreated >= LSN_CREATE_INTERVAL_SECONDS * 1000) {
576
- await createCheckpoint(this.client, this.defaultDb, this.checkpointStreamId);
635
+ await this.checkpointImplementation.createBatchCheckpoint();
577
636
  lastCheckpointCreated = performance.now();
578
637
  }
579
638
  batchesSeen += 1;
@@ -583,14 +642,15 @@ export class MongoSnapshotter {
583
642
  const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
584
643
 
585
644
  if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
586
- const checkpointId = changeDocument.documentKey._id as string | mongo.ObjectId;
587
- if (!this.checkpointStreamId.equals(checkpointId)) {
645
+ const kind = this.checkpointImplementation.event.observe(changeDocument);
646
+ if (kind != 'own-barrier') {
647
+ // Standalone events still feed the implementation's coordinate via
648
+ // event.observe above; we only resolve on our own barrier.
588
649
  continue;
589
650
  }
590
- return new MongoLSN({
591
- timestamp: changeDocument.clusterTime!,
592
- resume_token: changeDocument._id
593
- }).comparable;
651
+ // Observing our own barrier has set the coordinate (the barrier event
652
+ // carries the sentinel counter directly), so the LSN is ready.
653
+ return this.checkpointImplementation.event.lsn(changeDocument);
594
654
  }
595
655
 
596
656
  eventsSeen += 1;
@@ -648,7 +708,12 @@ export class MongoSnapshotter {
648
708
  }
649
709
  }
650
710
 
651
- const nsFilter = multipleDatabases
711
+ // DocumentDB always opens a cluster-level change stream, even in single-database
712
+ // mode, so a coll-only filter would match same-named collections in other
713
+ // databases of the cluster. Filter on the full namespace whenever the stream
714
+ // is cluster-scoped. See ChangeStream.getSourceNamespaceFilters for details.
715
+ const useFullNamespaceFilter = this.isDocumentDb || multipleDatabases;
716
+ const nsFilter = useFullNamespaceFilter
652
717
  ? { ns: { $in: inFilters } }
653
718
  : { 'ns.coll': { $in: inFilters.map((ns) => ns.coll) } };
654
719
  if (regexFilters.length > 0) {
@@ -665,12 +730,15 @@ export class MongoSnapshotter {
665
730
  signal?: AbortSignal;
666
731
  tracer?: PerformanceTracer<'changestream'>;
667
732
  }): AsyncIterableIterator<ChangeStreamBatch> {
668
- const lastLsn = options.lsn ? MongoLSN.fromSerialized(options.lsn) : null;
669
- const startAfter = lastLsn?.timestamp;
670
- const resumeAfter = lastLsn?.resumeToken;
733
+ const position = options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null;
734
+ const startAfter = position?.startAfter ?? undefined;
735
+ const resumeAfter = position?.resumeAfter ?? undefined;
671
736
 
672
737
  let fullDocument: 'required' | 'updateLookup';
673
- if (this.usePostImages) {
738
+ if (this.isDocumentDb) {
739
+ // DocumentDB does not support changeStreamPreAndPostImages, so 'required' won't work.
740
+ fullDocument = 'updateLookup';
741
+ } else if (this.usePostImages) {
674
742
  // 'read_only' or 'auto_configure'
675
743
  // Configuration happens during snapshot, or when we see new
676
744
  // collections.
@@ -679,27 +747,32 @@ export class MongoSnapshotter {
679
747
  fullDocument = 'updateLookup';
680
748
  }
681
749
  const streamOptions: mongo.ChangeStreamOptions & mongo.Document = {
682
- showExpandedEvents: true,
683
750
  fullDocument
684
751
  };
685
- const pipeline: mongo.Document[] = [
686
- { $changeStream: streamOptions },
687
- { $match: options.filters.$match },
688
- { $changeStreamSplitLargeEvent: {} }
689
- ];
752
+ if (!this.isDocumentDb) {
753
+ // DocumentDB does not support showExpandedEvents.
754
+ streamOptions.showExpandedEvents = true;
755
+ }
756
+ const pipeline: mongo.Document[] = [{ $changeStream: streamOptions }, { $match: options.filters.$match }];
757
+ if (!this.isDocumentDb) {
758
+ // DocumentDB does not support $changeStreamSplitLargeEvent.
759
+ pipeline.push({ $changeStreamSplitLargeEvent: {} });
760
+ }
690
761
 
691
762
  // Only one of these options can be supplied at a time.
692
763
  if (resumeAfter) {
693
764
  streamOptions.resumeAfter = resumeAfter;
694
- } else {
765
+ } else if (startAfter != null) {
695
766
  // Legacy: We don't persist lsns without resumeTokens anymore, but we do still handle the
696
- // case if we have an old one.
767
+ // case if we have an old one. The sentinel implementation never produces a startAfter,
768
+ // and a fresh DocumentDB stream opens from "now" with neither option set.
697
769
  streamOptions.startAtOperationTime = startAfter;
698
770
  }
699
771
 
700
772
  let watchDb: mongo.Db;
701
- if (options.filters.multipleDatabases) {
702
- // Requires readAnyDatabase@admin on Atlas
773
+ if (this.isDocumentDb || options.filters.multipleDatabases) {
774
+ // DocumentDB only supports cluster-level change streams.
775
+ // Requires readAnyDatabase@admin on Atlas.
703
776
  watchDb = this.client.db('admin');
704
777
  streamOptions.allChangesForCluster = true;
705
778
  } else {
@@ -707,9 +780,11 @@ export class MongoSnapshotter {
707
780
  watchDb = this.defaultDb;
708
781
  }
709
782
 
783
+ const maxAwaitTimeMS = options.maxAwaitTimeMS ?? this.maxAwaitTimeMS;
784
+
710
785
  return rawChangeStream(watchDb, pipeline, {
711
786
  batchSize: options.batchSize ?? this.snapshotChunkLength,
712
- maxAwaitTimeMS: options.maxAwaitTimeMS ?? this.maxAwaitTimeMS,
787
+ maxAwaitTimeMS,
713
788
  maxTimeMS: this.changeStreamTimeout,
714
789
  signal: options.signal,
715
790
  logger: this.logger,