@powersync/service-module-mongodb 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/CHANGELOG.md +153 -5
  2. package/dist/api/MongoRouteAPIAdapter.d.ts +4 -2
  3. package/dist/api/MongoRouteAPIAdapter.js +23 -32
  4. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  5. package/dist/common/SentinelLSN.d.ts +37 -0
  6. package/dist/common/SentinelLSN.js +59 -0
  7. package/dist/common/SentinelLSN.js.map +1 -0
  8. package/dist/module/MongoModule.js +2 -1
  9. package/dist/module/MongoModule.js.map +1 -1
  10. package/dist/replication/ChangeStream.d.ts +24 -0
  11. package/dist/replication/ChangeStream.js +203 -67
  12. package/dist/replication/ChangeStream.js.map +1 -1
  13. package/dist/replication/ChangeStreamReplicationJob.js +3 -2
  14. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  15. package/dist/replication/ChangeStreamReplicator.js +1 -1
  16. package/dist/replication/ChangeStreamReplicator.js.map +1 -1
  17. package/dist/replication/MongoRelation.d.ts +36 -1
  18. package/dist/replication/MongoRelation.js +133 -18
  19. package/dist/replication/MongoRelation.js.map +1 -1
  20. package/dist/replication/MongoSnapshotter.d.ts +9 -1
  21. package/dist/replication/MongoSnapshotter.js +129 -67
  22. package/dist/replication/MongoSnapshotter.js.map +1 -1
  23. package/dist/replication/RawChangeStream.d.ts +4 -1
  24. package/dist/replication/RawChangeStream.js +6 -10
  25. package/dist/replication/RawChangeStream.js.map +1 -1
  26. package/dist/replication/checkpoints/CheckpointImplementation.d.ts +135 -0
  27. package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
  28. package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
  29. package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +58 -0
  30. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +224 -0
  31. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
  32. package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +27 -0
  33. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +116 -0
  34. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
  35. package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
  36. package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
  37. package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
  38. package/dist/replication/replication-utils.d.ts +6 -0
  39. package/dist/replication/replication-utils.js +19 -2
  40. package/dist/replication/replication-utils.js.map +1 -1
  41. package/dist/types/types.d.ts +7 -2
  42. package/dist/types/types.js +20 -2
  43. package/dist/types/types.js.map +1 -1
  44. package/package.json +11 -21
  45. package/src/api/MongoRouteAPIAdapter.ts +28 -39
  46. package/src/common/SentinelLSN.ts +78 -0
  47. package/src/module/MongoModule.ts +2 -1
  48. package/src/replication/ChangeStream.ts +223 -78
  49. package/src/replication/ChangeStreamReplicationJob.ts +3 -2
  50. package/src/replication/ChangeStreamReplicator.ts +1 -1
  51. package/src/replication/MongoRelation.ts +153 -21
  52. package/src/replication/MongoSnapshotter.ts +148 -73
  53. package/src/replication/RawChangeStream.ts +27 -32
  54. package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
  55. package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +269 -0
  56. package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +145 -0
  57. package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
  58. package/src/replication/replication-utils.ts +23 -2
  59. package/src/types/types.ts +27 -2
  60. package/test/DOCUMENTDB_TESTING.md +115 -0
  61. package/test/src/DatabaseType.ts +25 -0
  62. package/test/src/change_stream.test.ts +180 -71
  63. package/test/src/change_stream_utils.ts +117 -29
  64. package/test/src/checkpoint_retry.test.ts +134 -0
  65. package/test/src/config.test.ts +34 -0
  66. package/test/src/documentdb_helpers.test.ts +124 -0
  67. package/test/src/documentdb_mode.test.ts +1039 -0
  68. package/test/src/mongo_test.test.ts +15 -5
  69. package/test/src/raw_change_stream.test.ts +147 -126
  70. package/test/src/resume.test.ts +2 -2
  71. package/test/src/resume_token.test.ts +30 -0
  72. package/test/src/slow_tests.test.ts +4 -1
  73. package/test/src/test-timeouts.ts +23 -0
  74. package/test/src/util.ts +14 -32
  75. package/tsconfig.json +1 -4
  76. package/tsconfig.scripts.json +1 -2
  77. package/tsconfig.tsbuildinfo +1 -1
@@ -5,6 +5,7 @@ import {
5
5
  createCoreReplicationMetrics,
6
6
  initializeCoreReplicationMetrics,
7
7
  InternalOpId,
8
+ isBatchEnd,
8
9
  LEGACY_STORAGE_VERSION,
9
10
  OplogEntry,
10
11
  ProtocolOpId,
@@ -19,9 +20,15 @@ import {
19
20
  } from '@powersync/service-core';
20
21
  import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
21
22
 
23
+ import { SentinelLSN } from '@module/common/SentinelLSN.js';
22
24
  import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
23
25
  import { MongoManager } from '@module/replication/MongoManager.js';
24
- import { createCheckpoint, STANDALONE_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
26
+ import {
27
+ createCheckpoint,
28
+ createSentinelCheckpointLsn,
29
+ STANDALONE_CHECKPOINT_ID
30
+ } from '@module/replication/MongoRelation.js';
31
+ import { detectDocumentDb } from '@module/replication/replication-utils.js';
25
32
  import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
26
33
 
27
34
  import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
@@ -30,7 +37,7 @@ export class ChangeStreamTestContext {
30
37
  private _walStream?: ChangeStream;
31
38
  private abortController = new AbortController();
32
39
  private settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
33
- private syncRulesContent?: storage.PersistedSyncRulesContent;
40
+ private syncRulesContent?: storage.PersistedSyncConfigContent;
34
41
  public storage?: SyncRulesBucketStorage;
35
42
 
36
43
  /**
@@ -46,6 +53,11 @@ export class ChangeStreamTestContext {
46
53
  storageVersion?: number;
47
54
  mongoOptions?: Partial<NormalizedMongoConnectionConfig>;
48
55
  streamOptions?: Partial<ChangeStreamOptions>;
56
+ /**
57
+ * Optional override for tests that need to force a mode. By default the
58
+ * test context detects DocumentDB from the hello response.
59
+ */
60
+ documentDbMode?: boolean;
49
61
  }
50
62
  ) {
51
63
  const f = await factory({ doNotClear: options?.doNotClear });
@@ -56,15 +68,17 @@ export class ChangeStreamTestContext {
56
68
  }
57
69
 
58
70
  const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION;
71
+ const documentDbMode = options?.documentDbMode ?? (await detectDocumentDb(connectionManager.db));
59
72
 
60
- return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion);
73
+ return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion, documentDbMode);
61
74
  }
62
75
 
63
76
  constructor(
64
77
  public factory: BucketStorageFactory,
65
78
  public connectionManager: MongoManager,
66
79
  private streamOptions: Partial<ChangeStreamOptions> = {},
67
- private storageVersion: number = LEGACY_STORAGE_VERSION
80
+ private storageVersion: number = LEGACY_STORAGE_VERSION,
81
+ private documentDbMode: boolean = false
68
82
  ) {
69
83
  createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
70
84
  initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
@@ -101,26 +115,37 @@ export class ChangeStreamTestContext {
101
115
  }
102
116
 
103
117
  async updateSyncRules(content: string) {
104
- const syncRules = await this.factory.updateSyncRules(
118
+ const replicationStream = await this.factory.updateSyncRules(
105
119
  updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
106
120
  );
107
- this.syncRulesContent = syncRules;
108
- this.storage = this.factory.getInstance(syncRules);
121
+ this.syncRulesContent = replicationStream.syncConfigContent[0];
122
+ this.storage = this.factory.getInstance(replicationStream);
109
123
  return this.storage!;
110
124
  }
111
125
 
112
126
  async loadNextSyncRules() {
113
- const syncRules = await this.factory.getNextSyncRulesContent();
114
- if (syncRules == null) {
127
+ const syncConfig = await this.factory.getDeployingSyncConfig();
128
+ if (syncConfig == null) {
115
129
  throw new Error(`Next sync config not available`);
116
130
  }
117
131
 
118
- this.syncRulesContent = syncRules;
119
- this.storage = this.factory.getInstance(syncRules);
132
+ this.syncRulesContent = syncConfig.content;
133
+ this.storage = syncConfig.storage;
134
+ return this.storage!;
135
+ }
136
+
137
+ async loadActiveSyncRules() {
138
+ const syncConfig = await this.factory.getActiveSyncConfig();
139
+ if (syncConfig == null) {
140
+ throw new Error(`Active sync config not found`);
141
+ }
142
+
143
+ this.syncRulesContent = syncConfig.content;
144
+ this.storage = syncConfig.storage;
120
145
  return this.storage!;
121
146
  }
122
147
 
123
- private getSyncRulesContent(): storage.PersistedSyncRulesContent {
148
+ private getSyncConfigContent(): storage.PersistedSyncConfigContent {
124
149
  if (this.syncRulesContent == null) {
125
150
  throw new Error('Sync config not configured - call updateSyncRules() first');
126
151
  }
@@ -143,6 +168,7 @@ export class ChangeStreamTestContext {
143
168
  // a long time to abort the stream.
144
169
  maxAwaitTimeMS: this.streamOptions?.maxAwaitTimeMS ?? 200,
145
170
  snapshotChunkLength: this.streamOptions?.snapshotChunkLength,
171
+ keepaliveIntervalMs: this.streamOptions?.keepaliveIntervalMs,
146
172
  storageHooks: this.streamOptions?.storageHooks,
147
173
  snapshotHooks: this.streamOptions?.snapshotHooks,
148
174
  logger: this.streamOptions?.logger
@@ -170,7 +196,24 @@ export class ChangeStreamTestContext {
170
196
  * would result in inconsistent data.
171
197
  */
172
198
  async markSnapshotConsistent() {
173
- const checkpoint = await createCheckpoint(this.client, this.db, STANDALONE_CHECKPOINT_ID);
199
+ let checkpoint: string;
200
+ if (this.documentDbMode) {
201
+ const sentinelCheckpoint = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(this.client, this.db));
202
+ const status = await this.storage!.getStatus();
203
+ const resumeFrom = status.resumeLsn;
204
+ const resumeToken = resumeFrom ? SentinelLSN.fromSerialized(resumeFrom).resumeToken : null;
205
+
206
+ // This helper artificially marks the snapshot as consistent without
207
+ // waiting for the stream to observe the sentinel. Keep the sentinel as the
208
+ // comparable position, but carry forward the existing snapshot resume
209
+ // token so later DocumentDB streaming still resumes from a real token.
210
+ checkpoint = new SentinelLSN({
211
+ sentinel: sentinelCheckpoint.sentinel,
212
+ resume_token: resumeToken
213
+ }).comparable;
214
+ } else {
215
+ checkpoint = await createCheckpoint(this.db, STANDALONE_CHECKPOINT_ID);
216
+ }
174
217
 
175
218
  await using writer = await this.storage!.createWriter(test_utils.BATCH_OPTIONS);
176
219
  await writer.keepalive(checkpoint);
@@ -197,37 +240,65 @@ export class ChangeStreamTestContext {
197
240
 
198
241
  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
199
242
  let checkpoint = await this.getCheckpoint(options);
200
- const syncRules = this.getSyncRulesContent();
201
- const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start));
243
+ const syncConfigContent = this.getSyncConfigContent();
244
+ const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncConfigContent, bucket, start));
202
245
  return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
203
246
  }
204
247
 
205
248
  async getBucketData(bucket: string, start?: ProtocolOpId | InternalOpId | undefined, options?: { timeout?: number }) {
249
+ const checkpoint = await this.getCheckpoint(options);
250
+ return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
251
+ }
252
+
253
+ async getBucketDataAtLatestCheckpoint(bucket: string, start?: ProtocolOpId | InternalOpId | undefined) {
254
+ if (this.storage == null) {
255
+ throw new Error('updateSyncRules() first');
256
+ }
257
+
258
+ const checkpoint = await this.storage.getCheckpoint();
259
+ return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
260
+ }
261
+
262
+ async getBucketDataAtCheckpoint(
263
+ bucket: string,
264
+ checkpoint: ReplicationCheckpoint,
265
+ start?: ProtocolOpId | InternalOpId | undefined
266
+ ) {
206
267
  start ??= 0n;
207
268
  if (typeof start == 'string') {
208
269
  start = BigInt(start);
209
270
  }
210
- const syncRules = this.getSyncRulesContent();
211
- const checkpoint = await this.getCheckpoint(options);
212
- let map = [bucketRequest(syncRules, bucket, start)];
271
+ const syncConfigContent = this.getSyncConfigContent();
272
+ let map = [bucketRequest(syncConfigContent, bucket, start)];
213
273
  let data: OplogEntry[] = [];
214
274
  while (true) {
215
275
  const batch = this.storage!.getBucketDataBatch(checkpoint, map);
216
276
 
217
- const batches = await test_utils.fromAsync(batch);
218
- data = data.concat(batches[0]?.chunkData.data ?? []);
219
- if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
277
+ const chunks = await test_utils.fromAsync(batch);
278
+ if (chunks.length == 0) {
220
279
  break;
221
280
  }
222
- map = [bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))];
281
+ for (let chunk of chunks) {
282
+ if (isBatchEnd(chunk)) {
283
+ if (!chunk.hasMore) {
284
+ return data;
285
+ }
286
+ } else {
287
+ data = data.concat(chunk.chunkData.data ?? []);
288
+ map = [bucketRequest(syncConfigContent, bucket, BigInt(chunk.chunkData.next_after))];
289
+ if (!chunk.chunkData.has_more) {
290
+ return data;
291
+ }
292
+ }
293
+ }
223
294
  }
224
295
  return data;
225
296
  }
226
297
 
227
298
  async getChecksums(buckets: string[], options?: { timeout?: number }): Promise<utils.ChecksumMap> {
228
299
  let checkpoint = await this.getCheckpoint(options);
229
- const syncRules = this.getSyncRulesContent();
230
- const versionedBuckets = buckets.map((bucket) => bucketRequest(syncRules, bucket, 0n));
300
+ const syncConfigContent = this.getSyncConfigContent();
301
+ const versionedBuckets = buckets.map((bucket) => bucketRequest(syncConfigContent, bucket, 0n));
231
302
  const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
232
303
 
233
304
  const unversioned: utils.ChecksumMap = new Map();
@@ -247,25 +318,42 @@ export async function getClientCheckpoint(
247
318
  client: mongo.MongoClient,
248
319
  db: mongo.Db,
249
320
  storageFactory: BucketStorageFactory,
250
- options?: { timeout?: number }
251
- ): Promise<InternalOpId> {
321
+ options?: { timeout?: number; documentDbMode?: boolean; storage?: SyncRulesBucketStorage }
322
+ ): Promise<ReplicationCheckpoint> {
252
323
  const start = Date.now();
253
- const lsn = await createCheckpoint(client, db, STANDALONE_CHECKPOINT_ID);
324
+ const documentDbMode = options?.documentDbMode ?? (await detectDocumentDb(db));
325
+
326
+ const lsn = documentDbMode
327
+ ? await createSentinelCheckpointLsn(client, db)
328
+ : await createCheckpoint(db, STANDALONE_CHECKPOINT_ID);
254
329
  // This old API needs a persisted checkpoint id.
255
330
  // Since we don't use LSNs anymore, the only way to get that is to wait.
256
331
 
257
332
  const timeout = options?.timeout ?? 50_000;
333
+ // DocumentDB: the streaming loop skips standalone checkpoint events while a
334
+ // batch barrier is pending (see ChangeStream.ts), so a single sentinel bump
335
+ // can be missed on an idle stream with no later event to advance the
336
+ // checkpoint. Periodically re-bump the sentinel so a standalone event
337
+ // eventually commits past `lsn`, mirroring getSnapshotLsn's retry loop.
338
+ const NUDGE_INTERVAL_MS = 1000;
339
+ let lastNudge = Date.now();
258
340
  let lastCp: ReplicationCheckpoint | null = null;
259
341
 
260
342
  while (Date.now() - start < timeout) {
261
- const storage = await storageFactory.getActiveStorage();
343
+ const storage = (await storageFactory.getActiveSyncConfig())?.storage;
262
344
  const cp = await storage?.getCheckpoint();
263
345
  if (cp != null) {
264
346
  lastCp = cp;
265
347
  if (cp.lsn && cp.lsn >= lsn) {
266
- return cp.checkpoint;
348
+ return cp;
267
349
  }
268
350
  }
351
+
352
+ if (documentDbMode && Date.now() - lastNudge >= NUDGE_INTERVAL_MS) {
353
+ await createSentinelCheckpointLsn(client, db);
354
+ lastNudge = Date.now();
355
+ }
356
+
269
357
  await new Promise((resolve) => setTimeout(resolve, 30));
270
358
  }
271
359
 
@@ -0,0 +1,134 @@
1
+ import { describe, expect, test } from 'vitest';
2
+
3
+ import { mongo } from '@powersync/lib-service-mongodb';
4
+
5
+ import { MongoLSN } from '@module/common/MongoLSN.js';
6
+ import { createCheckpoint } from '@module/replication/MongoRelation.js';
7
+ import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
8
+ import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
9
+
10
+ // This suite uses configureFailPoint to force retry timing. Azure DocumentDB
11
+ // does not support configureFailPoint.
12
+ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('checkpoint retryable writes', () => {
13
+ test('returns the persisted checkpoint clusterTime after a retryable write', { repeats: 1 }, async (ctx) => {
14
+ // This test relies on very specific timing:
15
+ //
16
+ // 1. _powersync_checkpoints findAndModify command starts and is sent to the server
17
+ // 2. command is written to the server
18
+ // 3. _powersync_checkpoints findAndModify command fails with a retryable error (e.g. connection timeout)
19
+ // 4. command is retried
20
+ // 5. another write lands on the server, increasing the server's operationTime
21
+ // 6. command succeeds with a new operationTime, but without actually writing again (server de-duplicates based on transaction id)
22
+ //
23
+ // It is quite difficult to simulate this, even with failCommand. We currently rely on triggering a socket timeout for the first command,
24
+ // with another write happening right after that.
25
+
26
+ const TIMEOUT = 200;
27
+ const INSERT_COUNT = 5;
28
+
29
+ const { db, client } = await connectMongoData({
30
+ monitorCommands: true,
31
+ socketTimeoutMS: TIMEOUT,
32
+ maxPoolSize: 1
33
+ });
34
+
35
+ await using _client = { [Symbol.asyncDispose]: async () => await client.close() };
36
+ await client.connect();
37
+ await using failCommand = await requireFailCommand(client, ctx);
38
+
39
+ await clearTestDb(db);
40
+
41
+ const checkpointId = new mongo.ObjectId();
42
+ const advanceClusterTime = db.collection('_advance_cluster_time');
43
+
44
+ const changeStream = db.watch([], { maxAwaitTimeMS: TIMEOUT * 0.8 });
45
+ await using _changeStream = { [Symbol.asyncDispose]: async () => await changeStream.close() };
46
+
47
+ const checkpointEventPromise = waitForCheckpointEvent(changeStream, checkpointId, INSERT_COUNT).catch((e) =>
48
+ console.error(e)
49
+ );
50
+
51
+ const commandsStarted: mongo.CommandStartedEvent[] = [];
52
+
53
+ client.on('commandStarted', (event) => {
54
+ commandsStarted.push(event);
55
+ });
56
+
57
+ const DEBUG_COMMANDS = false;
58
+ if (DEBUG_COMMANDS) {
59
+ // For debugging: Log all commands
60
+ client.on('commandStarted', (event) => {
61
+ console.log('commandStarted:', event);
62
+ });
63
+
64
+ client.on('commandSucceeded', (event) => {
65
+ console.log('commandSucceeded:', event);
66
+ });
67
+
68
+ client.on('commandFailed', (event) => {
69
+ console.log('commandFailed:', event);
70
+ });
71
+ }
72
+
73
+ await failCommand.configure({
74
+ mode: { times: 1 },
75
+ data: {
76
+ failCommands: ['commitTransaction', 'findAndModify'],
77
+ blockConnection: true,
78
+ blockTimeMS: TIMEOUT
79
+ }
80
+ });
81
+ const returnedLsnPromise = createCheckpoint(db, checkpointId);
82
+
83
+ for (let i = 0; i < INSERT_COUNT; i++) {
84
+ await advanceClusterTime.insertOne({ at: new Date() });
85
+ }
86
+
87
+ const returnedLsn = await returnedLsnPromise;
88
+ const checkpointEvent = await checkpointEventPromise;
89
+ if (checkpointEvent == null) {
90
+ throw new Error('change stream failed');
91
+ }
92
+
93
+ const eventLsn = new MongoLSN({
94
+ timestamp: checkpointEvent.clusterTime!,
95
+ resume_token: checkpointEvent._id
96
+ }).comparable;
97
+
98
+ if (DEBUG_COMMANDS) {
99
+ const commands = commandsStarted
100
+ .filter((c) => ['findAndModify', 'insert'].includes(c.commandName))
101
+ .map((c) => c.commandName);
102
+ // In the failure case, we had findAndModify, insert, findAndModify, potentially with more inserts in between.
103
+ // With updated behavior, we only have a single findAndModify.
104
+ console.log('Command order:', commands);
105
+ }
106
+
107
+ expect(eventLsn >= returnedLsn).toBe(true);
108
+ expect(checkpointEvent.clusterTime).toEqual(MongoLSN.fromSerialized(returnedLsn).timestamp);
109
+ });
110
+ });
111
+
112
+ async function waitForCheckpointEvent(
113
+ changeStream: mongo.ChangeStream,
114
+ checkpointId: mongo.ObjectId,
115
+ insertCount: number
116
+ ): Promise<mongo.ChangeStreamDocument | null> {
117
+ let lastEvent: mongo.ChangeStreamDocument | null = null;
118
+ let seenInserts = 0;
119
+ for await (const event of changeStream) {
120
+ if (
121
+ (event.operationType == 'insert' || event.operationType == 'update' || event.operationType == 'replace') &&
122
+ checkpointId.equals(event.documentKey._id)
123
+ ) {
124
+ lastEvent = event;
125
+ } else if (event.operationType == 'insert' && event.ns.coll === '_advance_cluster_time') {
126
+ seenInserts++;
127
+ if (seenInserts >= insertCount) {
128
+ // We finish when we hit the expected number of inserts, since the number of checkpoint events are unknown.
129
+ break;
130
+ }
131
+ }
132
+ }
133
+ return lastEvent;
134
+ }
@@ -0,0 +1,34 @@
1
+ import { normalizeConnectionConfig } from '@module/types/types.js';
2
+ import { describe, expect, test } from 'vitest';
3
+
4
+ const BASE_CONFIG = {
5
+ type: 'mongodb' as const,
6
+ uri: 'mongodb://localhost:27017/powersync_test'
7
+ };
8
+
9
+ describe('MongoDB connection config', () => {
10
+ test('defaults heartbeat_interval_seconds to 60 seconds', () => {
11
+ expect(normalizeConnectionConfig(BASE_CONFIG).heartbeat_interval_seconds).toBe(60);
12
+ });
13
+
14
+ test('defaults a null heartbeat_interval_seconds to 60 seconds', () => {
15
+ expect(
16
+ normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: null }).heartbeat_interval_seconds
17
+ ).toBe(60);
18
+ });
19
+
20
+ test('allows the MongoDB maximum heartbeat interval', () => {
21
+ expect(
22
+ normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: 60 }).heartbeat_interval_seconds
23
+ ).toBe(60);
24
+ });
25
+
26
+ test.each([0, 4, 61, Number.NaN, Number.POSITIVE_INFINITY])(
27
+ 'rejects invalid heartbeat_interval_seconds: %s',
28
+ (heartbeat_interval_seconds) => {
29
+ expect(() => normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds })).toThrow(
30
+ 'heartbeat_interval_seconds'
31
+ );
32
+ }
33
+ );
34
+ });
@@ -0,0 +1,124 @@
1
+ import { describe, expect, test } from 'vitest';
2
+
3
+ import { SentinelLSN } from '@module/common/SentinelLSN.js';
4
+ import { getEventTimestamp } from '@module/replication/checkpoints/CheckpointImplementation.js';
5
+ import { createSentinelCheckpointLsn, SENTINEL_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
6
+ import { CHECKPOINTS_COLLECTION } from '@module/replication/replication-utils.js';
7
+ import { mongo } from '@powersync/lib-service-mongodb';
8
+ import { connectMongoData } from './util.js';
9
+
10
+ describe('DocumentDB helpers', () => {
11
+ describe('getEventTimestamp', () => {
12
+ // getEventTimestamp is used by the timestamp checkpoint implementation (standard
13
+ // MongoDB): it prefers clusterTime and falls back to second-precision wallTime.
14
+ // The DocumentDB sentinel implementation does not use it — DocumentDB LSNs are
15
+ // sentinel-counter based, not timestamp based.
16
+
17
+ test('returns clusterTime when present', () => {
18
+ const clusterTime = mongo.Timestamp.fromBits(1, 1700000000);
19
+ const event = { clusterTime, wallTime: new Date('2024-01-01T00:00:00Z') } as any;
20
+ expect(getEventTimestamp(event)).toEqual(clusterTime);
21
+ });
22
+
23
+ test('falls back to second-precision wallTime when clusterTime is absent', () => {
24
+ const wallTime = new Date('2024-06-15T12:00:00.789Z');
25
+ const result = getEventTimestamp({ wallTime } as any);
26
+ expect(result.getHighBitsUnsigned()).toEqual(Math.floor(wallTime.getTime() / 1000));
27
+ // Increment is always 0 — wallTime only has second resolution in this conversion.
28
+ expect(result.getLowBitsUnsigned()).toEqual(0);
29
+ });
30
+
31
+ test('throws when neither clusterTime nor wallTime is present', () => {
32
+ expect(() => getEventTimestamp({} as any)).toThrow();
33
+ });
34
+ });
35
+
36
+ describe('DocumentDB detection', () => {
37
+ // Detection logic: hello.internal?.documentdb_versions != null
38
+ const isDocumentDb = (hello: any) => hello.internal?.documentdb_versions != null;
39
+
40
+ test('hello with documentdb_versions — detected as DocumentDB', () => {
41
+ const hello = {
42
+ isWritablePrimary: true,
43
+ msg: 'isdbgrid',
44
+ internal: {
45
+ documentdb_versions: ['1.111-0', '1.112.0', '12.1-1']
46
+ }
47
+ };
48
+ expect(isDocumentDb(hello)).toBe(true);
49
+ });
50
+
51
+ test('standard hello response — not DocumentDB', () => {
52
+ const hello = {
53
+ isWritablePrimary: true,
54
+ setName: 'rs0',
55
+ hosts: ['localhost:27017']
56
+ };
57
+ expect(isDocumentDb(hello)).toBe(false);
58
+ });
59
+ });
60
+
61
+ describe('sentinel checkpoint format', () => {
62
+ test('sentinel counter is seeded at a timestamp value on creation', { timeout: 30_000 }, async () => {
63
+ // If a consumer deletes the sentinel checkpoint document in their
64
+ // source database, the re-created counter must not restart below
65
+ // already-committed LSNs. createSentinelCheckpointLsn seeds new counters at
66
+ // the current epoch seconds shifted into the high 32 bits (resembling a
67
+ // MongoDB timestamp), so the coordinate jumps forward instead of resetting.
68
+ const { client, db: sharedDb } = await connectMongoData();
69
+ // Use an isolated database: other test files run in parallel against the
70
+ // shared test database and both bump and clear the sentinel checkpoint
71
+ // document, which would make these exact assertions racy.
72
+ const db = client.db(`${sharedDb.databaseName}_seed_test`);
73
+ const seed = (epochMs: number) => (BigInt(Math.floor(epochMs / 1000)) << 32n) + 1n;
74
+ try {
75
+ const before = Date.now();
76
+ const first = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(client, db));
77
+ const after = Date.now();
78
+
79
+ // Seeded at (epoch_seconds << 32) + 1, not at 1.
80
+ expect(first.sentinel).toBeGreaterThanOrEqual(seed(before));
81
+ expect(first.sentinel).toBeLessThanOrEqual(seed(after));
82
+
83
+ // Subsequent calls increment normally.
84
+ const second = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(client, db));
85
+ expect(second.sentinel).toEqual(first.sentinel + 1n);
86
+
87
+ // Simulate a consumer deleting the document after the counter has
88
+ // accumulated increments: the re-created counter resumes ahead. The seed
89
+ // has second granularity, and in production the document lives from
90
+ // initial sync onward, so re-creation always lands in a later second;
91
+ // wait for the clock to advance to reproduce that here.
92
+ await new Promise((resolve) => setTimeout(resolve, 1_100));
93
+ await db.collection(CHECKPOINTS_COLLECTION).deleteOne({ _id: SENTINEL_CHECKPOINT_ID as any });
94
+ const recreated = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(client, db));
95
+ expect(recreated.sentinel).toBeGreaterThan(second.sentinel);
96
+ } finally {
97
+ await db.dropDatabase().catch(() => {});
98
+ await client.close();
99
+ }
100
+ });
101
+
102
+ test('createSentinelCheckpointLsn stamps stream_id on the sentinel document', async () => {
103
+ // A batch barrier passes the stream's checkpointStreamId, which is stamped
104
+ // on the shared sentinel document so the stream can recognise its own
105
+ // barriers (and ignore other streams') by content.
106
+ const { client, db } = await connectMongoData();
107
+ try {
108
+ const barrierId = new mongo.ObjectId();
109
+ await createSentinelCheckpointLsn(client, db, barrierId);
110
+
111
+ const doc = await db.collection(CHECKPOINTS_COLLECTION).findOne({ _id: SENTINEL_CHECKPOINT_ID as any });
112
+ expect(barrierId.equals(doc!.stream_id)).true;
113
+ } finally {
114
+ await client.close();
115
+ }
116
+ });
117
+ });
118
+
119
+ // Note: the DocumentDB keepalive does not build an LSN from Date.now() — it bumps
120
+ // the shared sentinel via createSentinelCheckpointLsn and commits when the bump's
121
+ // own event is observed. The only Date.now()-derived value is the sentinel
122
+ // counter seed, which is covered by the 'sentinel counter is seeded at a
123
+ // timestamp value on creation' test above.
124
+ });