@powersync/service-module-mongodb 0.18.2 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/dist/api/MongoRouteAPIAdapter.d.ts +2 -0
  3. package/dist/api/MongoRouteAPIAdapter.js +23 -32
  4. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  5. package/dist/common/SentinelLSN.d.ts +37 -0
  6. package/dist/common/SentinelLSN.js +59 -0
  7. package/dist/common/SentinelLSN.js.map +1 -0
  8. package/dist/module/MongoModule.js +2 -1
  9. package/dist/module/MongoModule.js.map +1 -1
  10. package/dist/replication/ChangeStream.d.ts +23 -0
  11. package/dist/replication/ChangeStream.js +167 -47
  12. package/dist/replication/ChangeStream.js.map +1 -1
  13. package/dist/replication/ChangeStreamReplicationJob.js +2 -1
  14. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  15. package/dist/replication/MongoRelation.d.ts +36 -1
  16. package/dist/replication/MongoRelation.js +102 -6
  17. package/dist/replication/MongoRelation.js.map +1 -1
  18. package/dist/replication/MongoSnapshotter.d.ts +9 -0
  19. package/dist/replication/MongoSnapshotter.js +113 -42
  20. package/dist/replication/MongoSnapshotter.js.map +1 -1
  21. package/dist/replication/RawChangeStream.d.ts +12 -1
  22. package/dist/replication/RawChangeStream.js +23 -10
  23. package/dist/replication/RawChangeStream.js.map +1 -1
  24. package/dist/replication/checkpoints/CheckpointImplementation.d.ts +135 -0
  25. package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
  26. package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
  27. package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +58 -0
  28. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +224 -0
  29. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
  30. package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +27 -0
  31. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +116 -0
  32. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
  33. package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
  34. package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
  35. package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
  36. package/dist/replication/replication-utils.d.ts +6 -0
  37. package/dist/replication/replication-utils.js +19 -2
  38. package/dist/replication/replication-utils.js.map +1 -1
  39. package/dist/types/types.d.ts +5 -0
  40. package/dist/types/types.js +20 -2
  41. package/dist/types/types.js.map +1 -1
  42. package/package.json +9 -9
  43. package/src/api/MongoRouteAPIAdapter.ts +26 -37
  44. package/src/common/SentinelLSN.ts +78 -0
  45. package/src/module/MongoModule.ts +2 -1
  46. package/src/replication/ChangeStream.ts +184 -55
  47. package/src/replication/ChangeStreamReplicationJob.ts +2 -1
  48. package/src/replication/MongoRelation.ts +124 -14
  49. package/src/replication/MongoSnapshotter.ts +132 -47
  50. package/src/replication/RawChangeStream.ts +54 -32
  51. package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
  52. package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +269 -0
  53. package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +145 -0
  54. package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
  55. package/src/replication/replication-utils.ts +23 -2
  56. package/src/types/types.ts +27 -2
  57. package/test/DOCUMENTDB_TESTING.md +115 -0
  58. package/test/src/DatabaseType.ts +25 -0
  59. package/test/src/change_stream.test.ts +97 -65
  60. package/test/src/change_stream_utils.ts +99 -11
  61. package/test/src/checkpoint_retry.test.ts +5 -2
  62. package/test/src/config.test.ts +34 -0
  63. package/test/src/documentdb_helpers.test.ts +124 -0
  64. package/test/src/documentdb_mode.test.ts +1040 -0
  65. package/test/src/mongo_test.test.ts +15 -5
  66. package/test/src/raw_change_stream.test.ts +209 -125
  67. package/test/src/resume_token.test.ts +30 -0
  68. package/test/src/slow_tests.test.ts +4 -1
  69. package/test/src/test-timeouts.ts +23 -0
  70. package/test/src/util.ts +1 -2
  71. package/tsconfig.tsbuildinfo +1 -1
@@ -9,6 +9,8 @@ import { test_utils } from '@powersync/service-core-tests';
9
9
  import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
10
10
  import { PostImagesOption } from '@module/types/types.js';
11
11
  import { ChangeStreamTestContext } from './change_stream_utils.js';
12
+ import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
13
+ import { testTimeout } from './test-timeouts.js';
12
14
  import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
13
15
 
14
16
  const BASIC_SYNC_RULES = `
@@ -19,7 +21,7 @@ bucket_definitions:
19
21
  `;
20
22
 
21
23
  describe('change stream', () => {
22
- describeWithStorage({ timeout: 20_000 }, defineChangeStreamTests);
24
+ describeWithStorage({ timeout: testTimeout(20_000, { cloudOverride: 40_000 }) }, defineChangeStreamTests);
23
25
  });
24
26
 
25
27
  function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
@@ -28,7 +30,8 @@ function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTest
28
30
  const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
29
31
  return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
30
32
  };
31
- test('replicating basic values', async () => {
33
+ // DocumentDB does not support changeStreamPreAndPostImages, which this test requires via postImages: READ_ONLY.
34
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating basic values', async () => {
32
35
  await using context = await openContext({
33
36
  mongoOptions: { postImages: PostImagesOption.READ_ONLY }
34
37
  });
@@ -74,7 +77,10 @@ bucket_definitions:
74
77
  - SELECT _id as id, description, num FROM "test_%"`);
75
78
 
76
79
  await db.createCollection('test_data', {
77
- changeStreamPreAndPostImages: { enabled: false }
80
+ // DocumentDB does not support changeStreamPreAndPostImages, even when
81
+ // explicitly disabled. This test only needs a collection to exercise
82
+ // fatal error recovery, so omit the option in DocumentDB mode.
83
+ ...(DATABASE_TYPE == DatabaseType.MONGODB ? { changeStreamPreAndPostImages: { enabled: false } } : {})
78
84
  });
79
85
  const collection = db.collection('test_data');
80
86
 
@@ -152,7 +158,7 @@ bucket_definitions:
152
158
  await context.updateSyncRules(BASIC_SYNC_RULES);
153
159
 
154
160
  await db.createCollection('test_data', {
155
- changeStreamPreAndPostImages: { enabled: false }
161
+ changeStreamPreAndPostImages: DATABASE_TYPE == DatabaseType.MONGODB ? { enabled: false } : undefined
156
162
  });
157
163
  testId = new mongo.ObjectId();
158
164
  collection = db.collection('test_data');
@@ -167,7 +173,9 @@ bucket_definitions:
167
173
  expect(test_utils.reduceBucket(data).slice(1)).toEqual([]);
168
174
  });
169
175
 
170
- test('updateLookup - no fullDocument available', async () => {
176
+ // DocumentDB always materializes a write-time post-image for updateLookup, so the 'fullDocument unavailable'
177
+ // case this test exercises (update treated as a remove) cannot occur there.
178
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('updateLookup - no fullDocument available', async () => {
171
179
  await using context = await openContext({
172
180
  mongoOptions: { postImages: PostImagesOption.OFF }
173
181
  });
@@ -179,6 +187,9 @@ bucket_definitions:
179
187
  - SELECT _id as id, description, num FROM "test_data"`);
180
188
 
181
189
  await db.createCollection('test_data', {
190
+ // DocumentDB does not support changeStreamPreAndPostImages, even when
191
+ // explicitly disabled. This test only needs a collection to exercise
192
+ // fatal error recovery, so omit the option in DocumentDB mode.
182
193
  changeStreamPreAndPostImages: { enabled: false }
183
194
  });
184
195
  const collection = db.collection('test_data');
@@ -211,7 +222,8 @@ bucket_definitions:
211
222
  ]);
212
223
  });
213
224
 
214
- test('postImages - autoConfigure', async () => {
225
+ // DocumentDB does not support changeStreamPreAndPostImages (post-images).
226
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('postImages - autoConfigure', async () => {
215
227
  // Similar to the above test, but with postImages enabled.
216
228
  // This resolves the consistency issue.
217
229
  await using context = await openContext({
@@ -259,7 +271,8 @@ bucket_definitions:
259
271
  ]);
260
272
  });
261
273
 
262
- test('postImages - on', async () => {
274
+ // DocumentDB does not support changeStreamPreAndPostImages (post-images).
275
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('postImages - on', async () => {
263
276
  // Similar to postImages - autoConfigure, but does not auto-configure.
264
277
  // changeStreamPreAndPostImages must be manually configured.
265
278
  await using context = await openContext({
@@ -408,7 +421,9 @@ bucket_definitions:
408
421
  ]);
409
422
  });
410
423
 
411
- test('replicating dropCollection', async () => {
424
+ // Collection drop events are not currently replicated on DocumentDB: they are not delivered through the
425
+ // cluster-level change stream the way standard MongoDB delivers them, so the drop is not applied as a remove.
426
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating dropCollection', async () => {
412
427
  await using context = await openContext();
413
428
  const { db } = context;
414
429
  const syncRuleContent = `
@@ -440,7 +455,9 @@ bucket_definitions:
440
455
  ]);
441
456
  });
442
457
 
443
- test('replicating renameCollection', async () => {
458
+ // Collection rename events are not currently replicated on DocumentDB: they are not delivered through the
459
+ // cluster-level change stream the way standard MongoDB delivers them.
460
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating renameCollection', async () => {
444
461
  await using context = await openContext();
445
462
  const { db } = context;
446
463
  const syncRuleContent = `
@@ -471,7 +488,9 @@ bucket_definitions:
471
488
  ]);
472
489
  });
473
490
 
474
- test.runIf(supportsConcurrentSnapshots)(
491
+ // Concurrent-snapshot regression test driven by collection drop/recreate DDL events, which DocumentDB does
492
+ // not deliver through the cluster-level change stream the way standard MongoDB does.
493
+ test.runIf(supportsConcurrentSnapshots && DATABASE_TYPE != DatabaseType.DOCUMENTDB)(
475
494
  'collection recreated while queued snapshot is waiting does not stall checkpoints',
476
495
  async () => {
477
496
  // This is a regression test for a specific timing issue in concurrent snapshot logic.
@@ -655,12 +674,14 @@ bucket_definitions:
655
674
 
656
675
  // We need at least 1 commit.
657
676
  expect(commitCount).toBeGreaterThan(0);
658
- // The previous implementation greated 1 commit per checkpoint, which is bad for performance.
677
+ // The previous implementation created 1 commit per checkpoint, which is bad for performance.
659
678
  // We expect a small number here - typically 2-10, but allow for anything less than the total number of checkpoints.
660
679
  expect(commitCount).toBeLessThan(checkpointCount + 1);
661
680
  });
662
681
 
663
- test('large record', async () => {
682
+ test
683
+ // DocumentDBDB does not support changeStreamSplitLargeEvent
684
+ .skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('large record', async () => {
664
685
  // Test a large update.
665
686
 
666
687
  // Without $changeStreamSplitLargeEvent, we get this error:
@@ -732,75 +753,83 @@ bucket_definitions:
732
753
  expect(data).toMatchObject([]);
733
754
  });
734
755
 
735
- test('postImages - new collection with postImages enabled', async () => {
736
- await using context = await openContext({
737
- mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
738
- });
739
- const { db } = context;
740
- await context.updateSyncRules(`
756
+ // DocumentDB does not support changeStreamPreAndPostImages (post-images).
757
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
758
+ 'postImages - new collection with postImages enabled',
759
+ async () => {
760
+ await using context = await openContext({
761
+ mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
762
+ });
763
+ const { db } = context;
764
+ await context.updateSyncRules(`
741
765
  bucket_definitions:
742
766
  global:
743
767
  data:
744
768
  - SELECT _id as id, description FROM "test_%"`);
745
769
 
746
- await context.replicateSnapshot();
770
+ await context.replicateSnapshot();
747
771
 
748
- await db.createCollection('test_data', {
749
- // enabled: true here - everything should work
750
- changeStreamPreAndPostImages: { enabled: true }
751
- });
752
- const collection = db.collection('test_data');
753
- const result = await collection.insertOne({ description: 'test1' });
754
- const test_id = result.insertedId;
755
- await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
772
+ await db.createCollection('test_data', {
773
+ // enabled: true here - everything should work
774
+ changeStreamPreAndPostImages: { enabled: true }
775
+ });
776
+ const collection = db.collection('test_data');
777
+ const result = await collection.insertOne({ description: 'test1' });
778
+ const test_id = result.insertedId;
779
+ await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
756
780
 
757
- context.startStreaming();
781
+ context.startStreaming();
758
782
 
759
- const data = await context.getBucketData('global[]');
760
- if (data.length == 3) {
761
- expect(data).toMatchObject([
762
- // An extra op here, since this triggers a snapshot in addition to getting the event.
763
- test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
764
- test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
765
- test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
766
- ]);
767
- } else {
768
- expect(data).toMatchObject([
769
- test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
770
- test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
771
- ]);
783
+ const data = await context.getBucketData('global[]');
784
+ if (data.length == 3) {
785
+ expect(data).toMatchObject([
786
+ // An extra op here, since this triggers a snapshot in addition to getting the event.
787
+ test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
788
+ test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
789
+ test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
790
+ ]);
791
+ } else {
792
+ expect(data).toMatchObject([
793
+ test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
794
+ test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
795
+ ]);
796
+ }
772
797
  }
773
- });
798
+ );
774
799
 
775
- test('postImages - new collection with postImages disabled', async () => {
776
- await using context = await openContext({
777
- mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
778
- });
779
- const { db } = context;
780
- await context.updateSyncRules(`
800
+ // DocumentDB does not support changeStreamPreAndPostImages (post-images).
801
+ test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
802
+ 'postImages - new collection with postImages disabled',
803
+ async () => {
804
+ await using context = await openContext({
805
+ mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
806
+ });
807
+ const { db } = context;
808
+ await context.updateSyncRules(`
781
809
  bucket_definitions:
782
810
  global:
783
811
  data:
784
812
  - SELECT _id as id, description FROM "test_data%"`);
785
813
 
786
- await context.replicateSnapshot();
814
+ await context.replicateSnapshot();
787
815
 
788
- await db.createCollection('test_data', {
789
- // enabled: false here, but autoConfigure will enable it.
790
- // Unfortunately, that is too late, and replication must be restarted.
791
- changeStreamPreAndPostImages: { enabled: false }
792
- });
793
- const collection = db.collection('test_data');
794
- const result = await collection.insertOne({ description: 'test1' });
795
- const test_id = result.insertedId;
796
- await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
816
+ await db.createCollection('test_data', {
817
+ // enabled: false here, but autoConfigure will enable it.
818
+ // Unfortunately, that is too late, and replication must be restarted.
819
+ changeStreamPreAndPostImages: { enabled: false }
820
+ });
821
+ const collection = db.collection('test_data');
822
+ const result = await collection.insertOne({ description: 'test1' });
823
+ const test_id = result.insertedId;
824
+ await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
797
825
 
798
- context.startStreaming();
826
+ context.startStreaming();
799
827
 
800
- await expect(() => context.getBucketData('global[]')).rejects.toMatchObject({
801
- message: expect.stringContaining('stream was configured to require a post-image for all update events')
802
- });
803
- });
828
+ await expect(() => context.getBucketData('global[]')).rejects.toMatchObject({
829
+ message: expect.stringContaining('stream was configured to require a post-image for all update events')
830
+ });
831
+ }
832
+ );
804
833
 
805
834
  test('recover from error', async () => {
806
835
  await using context = await openContext();
@@ -812,7 +841,10 @@ bucket_definitions:
812
841
  - SELECT _id as id, description, num FROM "test_data"`);
813
842
 
814
843
  await db.createCollection('test_data', {
815
- changeStreamPreAndPostImages: { enabled: false }
844
+ // DocumentDB does not support changeStreamPreAndPostImages, even when
845
+ // explicitly disabled. This test only needs a collection to exercise
846
+ // fatal error recovery, so omit the option in DocumentDB mode.
847
+ changeStreamPreAndPostImages: DATABASE_TYPE == DatabaseType.MONGODB ? { enabled: false } : undefined
816
848
  });
817
849
 
818
850
  const collection = db.collection('test_data');
@@ -837,7 +869,7 @@ bucket_definitions:
837
869
  const error = (await syncRules?.getSyncConfigStatus())?.last_fatal_error;
838
870
  return error == null;
839
871
  },
840
- { timeout: 2_000 }
872
+ { timeout: testTimeout(2_000, { cloudOverride: 5_000 }) }
841
873
  );
842
874
  });
843
875
  }
@@ -5,6 +5,7 @@ import {
5
5
  createCoreReplicationMetrics,
6
6
  initializeCoreReplicationMetrics,
7
7
  InternalOpId,
8
+ isBatchEnd,
8
9
  LEGACY_STORAGE_VERSION,
9
10
  OplogEntry,
10
11
  ProtocolOpId,
@@ -19,9 +20,15 @@ import {
19
20
  } from '@powersync/service-core';
20
21
  import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
21
22
 
23
+ import { SentinelLSN } from '@module/common/SentinelLSN.js';
22
24
  import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
23
25
  import { MongoManager } from '@module/replication/MongoManager.js';
24
- import { createCheckpoint, STANDALONE_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
26
+ import {
27
+ createCheckpoint,
28
+ createSentinelCheckpointLsn,
29
+ STANDALONE_CHECKPOINT_ID
30
+ } from '@module/replication/MongoRelation.js';
31
+ import { detectDocumentDb } from '@module/replication/replication-utils.js';
25
32
  import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
26
33
 
27
34
  import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
@@ -46,6 +53,11 @@ export class ChangeStreamTestContext {
46
53
  storageVersion?: number;
47
54
  mongoOptions?: Partial<NormalizedMongoConnectionConfig>;
48
55
  streamOptions?: Partial<ChangeStreamOptions>;
56
+ /**
57
+ * Optional override for tests that need to force a mode. By default the
58
+ * test context detects DocumentDB from the hello response.
59
+ */
60
+ documentDbMode?: boolean;
49
61
  }
50
62
  ) {
51
63
  const f = await factory({ doNotClear: options?.doNotClear });
@@ -56,15 +68,17 @@ export class ChangeStreamTestContext {
56
68
  }
57
69
 
58
70
  const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION;
71
+ const documentDbMode = options?.documentDbMode ?? (await detectDocumentDb(connectionManager.db));
59
72
 
60
- return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion);
73
+ return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion, documentDbMode);
61
74
  }
62
75
 
63
76
  constructor(
64
77
  public factory: BucketStorageFactory,
65
78
  public connectionManager: MongoManager,
66
79
  private streamOptions: Partial<ChangeStreamOptions> = {},
67
- private storageVersion: number = LEGACY_STORAGE_VERSION
80
+ private storageVersion: number = LEGACY_STORAGE_VERSION,
81
+ private documentDbMode: boolean = false
68
82
  ) {
69
83
  createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
70
84
  initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
@@ -120,6 +134,17 @@ export class ChangeStreamTestContext {
120
134
  return this.storage!;
121
135
  }
122
136
 
137
+ async loadActiveSyncRules() {
138
+ const syncConfig = await this.factory.getActiveSyncConfig();
139
+ if (syncConfig == null) {
140
+ throw new Error(`Active sync config not found`);
141
+ }
142
+
143
+ this.syncRulesContent = syncConfig.content;
144
+ this.storage = syncConfig.storage;
145
+ return this.storage!;
146
+ }
147
+
123
148
  private getSyncConfigContent(): storage.PersistedSyncConfigContent {
124
149
  if (this.syncRulesContent == null) {
125
150
  throw new Error('Sync config not configured - call updateSyncRules() first');
@@ -143,6 +168,7 @@ export class ChangeStreamTestContext {
143
168
  // a long time to abort the stream.
144
169
  maxAwaitTimeMS: this.streamOptions?.maxAwaitTimeMS ?? 200,
145
170
  snapshotChunkLength: this.streamOptions?.snapshotChunkLength,
171
+ keepaliveIntervalMs: this.streamOptions?.keepaliveIntervalMs,
146
172
  storageHooks: this.streamOptions?.storageHooks,
147
173
  snapshotHooks: this.streamOptions?.snapshotHooks,
148
174
  logger: this.streamOptions?.logger
@@ -170,7 +196,24 @@ export class ChangeStreamTestContext {
170
196
  * would result in inconsistent data.
171
197
  */
172
198
  async markSnapshotConsistent() {
173
- const checkpoint = await createCheckpoint(this.client, this.db, STANDALONE_CHECKPOINT_ID);
199
+ let checkpoint: string;
200
+ if (this.documentDbMode) {
201
+ const sentinelCheckpoint = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(this.client, this.db));
202
+ const status = await this.storage!.getStatus();
203
+ const resumeFrom = status.resumeLsn;
204
+ const resumeToken = resumeFrom ? SentinelLSN.fromSerialized(resumeFrom).resumeToken : null;
205
+
206
+ // This helper artificially marks the snapshot as consistent without
207
+ // waiting for the stream to observe the sentinel. Keep the sentinel as the
208
+ // comparable position, but carry forward the existing snapshot resume
209
+ // token so later DocumentDB streaming still resumes from a real token.
210
+ checkpoint = new SentinelLSN({
211
+ sentinel: sentinelCheckpoint.sentinel,
212
+ resume_token: resumeToken
213
+ }).comparable;
214
+ } else {
215
+ checkpoint = await createCheckpoint(this.db, STANDALONE_CHECKPOINT_ID);
216
+ }
174
217
 
175
218
  await using writer = await this.storage!.createWriter(test_utils.BATCH_OPTIONS);
176
219
  await writer.keepalive(checkpoint);
@@ -203,23 +246,51 @@ export class ChangeStreamTestContext {
203
246
  }
204
247
 
205
248
  async getBucketData(bucket: string, start?: ProtocolOpId | InternalOpId | undefined, options?: { timeout?: number }) {
249
+ const checkpoint = await this.getCheckpoint(options);
250
+ return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
251
+ }
252
+
253
+ async getBucketDataAtLatestCheckpoint(bucket: string, start?: ProtocolOpId | InternalOpId | undefined) {
254
+ if (this.storage == null) {
255
+ throw new Error('updateSyncRules() first');
256
+ }
257
+
258
+ const checkpoint = await this.storage.getCheckpoint();
259
+ return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
260
+ }
261
+
262
+ async getBucketDataAtCheckpoint(
263
+ bucket: string,
264
+ checkpoint: ReplicationCheckpoint,
265
+ start?: ProtocolOpId | InternalOpId | undefined
266
+ ) {
206
267
  start ??= 0n;
207
268
  if (typeof start == 'string') {
208
269
  start = BigInt(start);
209
270
  }
210
271
  const syncConfigContent = this.getSyncConfigContent();
211
- const checkpoint = await this.getCheckpoint(options);
212
272
  let map = [bucketRequest(syncConfigContent, bucket, start)];
213
273
  let data: OplogEntry[] = [];
214
274
  while (true) {
215
275
  const batch = this.storage!.getBucketDataBatch(checkpoint, map);
216
276
 
217
- const batches = await test_utils.fromAsync(batch);
218
- data = data.concat(batches[0]?.chunkData.data ?? []);
219
- if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
277
+ const chunks = await test_utils.fromAsync(batch);
278
+ if (chunks.length == 0) {
220
279
  break;
221
280
  }
222
- map = [bucketRequest(syncConfigContent, bucket, BigInt(batches[0]!.chunkData.next_after))];
281
+ for (let chunk of chunks) {
282
+ if (isBatchEnd(chunk)) {
283
+ if (!chunk.hasMore) {
284
+ return data;
285
+ }
286
+ } else {
287
+ data = data.concat(chunk.chunkData.data ?? []);
288
+ map = [bucketRequest(syncConfigContent, bucket, BigInt(chunk.chunkData.next_after))];
289
+ if (!chunk.chunkData.has_more) {
290
+ return data;
291
+ }
292
+ }
293
+ }
223
294
  }
224
295
  return data;
225
296
  }
@@ -247,14 +318,25 @@ export async function getClientCheckpoint(
247
318
  client: mongo.MongoClient,
248
319
  db: mongo.Db,
249
320
  storageFactory: BucketStorageFactory,
250
- options?: { timeout?: number }
321
+ options?: { timeout?: number; documentDbMode?: boolean; storage?: SyncRulesBucketStorage }
251
322
  ): Promise<ReplicationCheckpoint> {
252
323
  const start = Date.now();
253
- const lsn = await createCheckpoint(client, db, STANDALONE_CHECKPOINT_ID);
324
+ const documentDbMode = options?.documentDbMode ?? (await detectDocumentDb(db));
325
+
326
+ const lsn = documentDbMode
327
+ ? await createSentinelCheckpointLsn(client, db)
328
+ : await createCheckpoint(db, STANDALONE_CHECKPOINT_ID);
254
329
  // This old API needs a persisted checkpoint id.
255
330
  // Since we don't use LSNs anymore, the only way to get that is to wait.
256
331
 
257
332
  const timeout = options?.timeout ?? 50_000;
333
+ // DocumentDB: the streaming loop skips standalone checkpoint events while a
334
+ // batch barrier is pending (see ChangeStream.ts), so a single sentinel bump
335
+ // can be missed on an idle stream with no later event to advance the
336
+ // checkpoint. Periodically re-bump the sentinel so a standalone event
337
+ // eventually commits past `lsn`, mirroring getSnapshotLsn's retry loop.
338
+ const NUDGE_INTERVAL_MS = 1000;
339
+ let lastNudge = Date.now();
258
340
  let lastCp: ReplicationCheckpoint | null = null;
259
341
 
260
342
  while (Date.now() - start < timeout) {
@@ -266,6 +348,12 @@ export async function getClientCheckpoint(
266
348
  return cp;
267
349
  }
268
350
  }
351
+
352
+ if (documentDbMode && Date.now() - lastNudge >= NUDGE_INTERVAL_MS) {
353
+ await createSentinelCheckpointLsn(client, db);
354
+ lastNudge = Date.now();
355
+ }
356
+
269
357
  await new Promise((resolve) => setTimeout(resolve, 30));
270
358
  }
271
359
 
@@ -4,9 +4,12 @@ import { mongo } from '@powersync/lib-service-mongodb';
4
4
 
5
5
  import { MongoLSN } from '@module/common/MongoLSN.js';
6
6
  import { createCheckpoint } from '@module/replication/MongoRelation.js';
7
+ import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
7
8
  import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
8
9
 
9
- describe('checkpoint retryable writes', () => {
10
+ // This suite uses configureFailPoint to force retry timing. Azure DocumentDB
11
+ // does not support configureFailPoint.
12
+ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('checkpoint retryable writes', () => {
10
13
  test('returns the persisted checkpoint clusterTime after a retryable write', { repeats: 1 }, async (ctx) => {
11
14
  // This test relies on very specific timing:
12
15
  //
@@ -75,7 +78,7 @@ describe('checkpoint retryable writes', () => {
75
78
  blockTimeMS: TIMEOUT
76
79
  }
77
80
  });
78
- const returnedLsnPromise = createCheckpoint(client, db, checkpointId);
81
+ const returnedLsnPromise = createCheckpoint(db, checkpointId);
79
82
 
80
83
  for (let i = 0; i < INSERT_COUNT; i++) {
81
84
  await advanceClusterTime.insertOne({ at: new Date() });
@@ -0,0 +1,34 @@
1
+ import { normalizeConnectionConfig } from '@module/types/types.js';
2
+ import { describe, expect, test } from 'vitest';
3
+
4
+ const BASE_CONFIG = {
5
+ type: 'mongodb' as const,
6
+ uri: 'mongodb://localhost:27017/powersync_test'
7
+ };
8
+
9
+ describe('MongoDB connection config', () => {
10
+ test('defaults heartbeat_interval_seconds to 60 seconds', () => {
11
+ expect(normalizeConnectionConfig(BASE_CONFIG).heartbeat_interval_seconds).toBe(60);
12
+ });
13
+
14
+ test('defaults a null heartbeat_interval_seconds to 60 seconds', () => {
15
+ expect(
16
+ normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: null }).heartbeat_interval_seconds
17
+ ).toBe(60);
18
+ });
19
+
20
+ test('allows the MongoDB maximum heartbeat interval', () => {
21
+ expect(
22
+ normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: 60 }).heartbeat_interval_seconds
23
+ ).toBe(60);
24
+ });
25
+
26
+ test.each([0, 4, 61, Number.NaN, Number.POSITIVE_INFINITY])(
27
+ 'rejects invalid heartbeat_interval_seconds: %s',
28
+ (heartbeat_interval_seconds) => {
29
+ expect(() => normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds })).toThrow(
30
+ 'heartbeat_interval_seconds'
31
+ );
32
+ }
33
+ );
34
+ });