@powersync/service-module-mongodb 0.18.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +2 -0
- package/dist/api/MongoRouteAPIAdapter.js +23 -32
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/common/SentinelLSN.d.ts +37 -0
- package/dist/common/SentinelLSN.js +59 -0
- package/dist/common/SentinelLSN.js.map +1 -0
- package/dist/module/MongoModule.js +2 -1
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +23 -0
- package/dist/replication/ChangeStream.js +167 -47
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +2 -1
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/MongoRelation.d.ts +36 -1
- package/dist/replication/MongoRelation.js +102 -6
- package/dist/replication/MongoRelation.js.map +1 -1
- package/dist/replication/MongoSnapshotter.d.ts +9 -0
- package/dist/replication/MongoSnapshotter.js +113 -42
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +12 -1
- package/dist/replication/RawChangeStream.js +23 -10
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/checkpoints/CheckpointImplementation.d.ts +135 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +58 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +224 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +27 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +116 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
- package/dist/replication/replication-utils.d.ts +6 -0
- package/dist/replication/replication-utils.js +19 -2
- package/dist/replication/replication-utils.js.map +1 -1
- package/dist/types/types.d.ts +5 -0
- package/dist/types/types.js +20 -2
- package/dist/types/types.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MongoRouteAPIAdapter.ts +26 -37
- package/src/common/SentinelLSN.ts +78 -0
- package/src/module/MongoModule.ts +2 -1
- package/src/replication/ChangeStream.ts +184 -55
- package/src/replication/ChangeStreamReplicationJob.ts +2 -1
- package/src/replication/MongoRelation.ts +124 -14
- package/src/replication/MongoSnapshotter.ts +132 -47
- package/src/replication/RawChangeStream.ts +54 -32
- package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
- package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +269 -0
- package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +145 -0
- package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
- package/src/replication/replication-utils.ts +23 -2
- package/src/types/types.ts +27 -2
- package/test/DOCUMENTDB_TESTING.md +115 -0
- package/test/src/DatabaseType.ts +25 -0
- package/test/src/change_stream.test.ts +97 -65
- package/test/src/change_stream_utils.ts +99 -11
- package/test/src/checkpoint_retry.test.ts +5 -2
- package/test/src/config.test.ts +34 -0
- package/test/src/documentdb_helpers.test.ts +124 -0
- package/test/src/documentdb_mode.test.ts +1040 -0
- package/test/src/mongo_test.test.ts +15 -5
- package/test/src/raw_change_stream.test.ts +209 -125
- package/test/src/resume_token.test.ts +30 -0
- package/test/src/slow_tests.test.ts +4 -1
- package/test/src/test-timeouts.ts +23 -0
- package/test/src/util.ts +1 -2
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -9,6 +9,8 @@ import { test_utils } from '@powersync/service-core-tests';
|
|
|
9
9
|
import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
|
|
10
10
|
import { PostImagesOption } from '@module/types/types.js';
|
|
11
11
|
import { ChangeStreamTestContext } from './change_stream_utils.js';
|
|
12
|
+
import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
|
|
13
|
+
import { testTimeout } from './test-timeouts.js';
|
|
12
14
|
import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
13
15
|
|
|
14
16
|
const BASIC_SYNC_RULES = `
|
|
@@ -19,7 +21,7 @@ bucket_definitions:
|
|
|
19
21
|
`;
|
|
20
22
|
|
|
21
23
|
describe('change stream', () => {
|
|
22
|
-
describeWithStorage({ timeout: 20_000 }, defineChangeStreamTests);
|
|
24
|
+
describeWithStorage({ timeout: testTimeout(20_000, { cloudOverride: 40_000 }) }, defineChangeStreamTests);
|
|
23
25
|
});
|
|
24
26
|
|
|
25
27
|
function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
|
|
@@ -28,7 +30,8 @@ function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTest
|
|
|
28
30
|
const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
|
|
29
31
|
return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
|
|
30
32
|
};
|
|
31
|
-
|
|
33
|
+
// DocumentDB does not support changeStreamPreAndPostImages, which this test requires via postImages: READ_ONLY.
|
|
34
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating basic values', async () => {
|
|
32
35
|
await using context = await openContext({
|
|
33
36
|
mongoOptions: { postImages: PostImagesOption.READ_ONLY }
|
|
34
37
|
});
|
|
@@ -74,7 +77,10 @@ bucket_definitions:
|
|
|
74
77
|
- SELECT _id as id, description, num FROM "test_%"`);
|
|
75
78
|
|
|
76
79
|
await db.createCollection('test_data', {
|
|
77
|
-
|
|
80
|
+
// DocumentDB does not support changeStreamPreAndPostImages, even when
|
|
81
|
+
// explicitly disabled. This test only needs a collection to exercise
|
|
82
|
+
// fatal error recovery, so omit the option in DocumentDB mode.
|
|
83
|
+
...(DATABASE_TYPE == DatabaseType.MONGODB ? { changeStreamPreAndPostImages: { enabled: false } } : {})
|
|
78
84
|
});
|
|
79
85
|
const collection = db.collection('test_data');
|
|
80
86
|
|
|
@@ -152,7 +158,7 @@ bucket_definitions:
|
|
|
152
158
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
153
159
|
|
|
154
160
|
await db.createCollection('test_data', {
|
|
155
|
-
changeStreamPreAndPostImages: { enabled: false }
|
|
161
|
+
changeStreamPreAndPostImages: DATABASE_TYPE == DatabaseType.MONGODB ? { enabled: false } : undefined
|
|
156
162
|
});
|
|
157
163
|
testId = new mongo.ObjectId();
|
|
158
164
|
collection = db.collection('test_data');
|
|
@@ -167,7 +173,9 @@ bucket_definitions:
|
|
|
167
173
|
expect(test_utils.reduceBucket(data).slice(1)).toEqual([]);
|
|
168
174
|
});
|
|
169
175
|
|
|
170
|
-
|
|
176
|
+
// DocumentDB always materializes a write-time post-image for updateLookup, so the 'fullDocument unavailable'
|
|
177
|
+
// case this test exercises (update treated as a remove) cannot occur there.
|
|
178
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('updateLookup - no fullDocument available', async () => {
|
|
171
179
|
await using context = await openContext({
|
|
172
180
|
mongoOptions: { postImages: PostImagesOption.OFF }
|
|
173
181
|
});
|
|
@@ -179,6 +187,9 @@ bucket_definitions:
|
|
|
179
187
|
- SELECT _id as id, description, num FROM "test_data"`);
|
|
180
188
|
|
|
181
189
|
await db.createCollection('test_data', {
|
|
190
|
+
// DocumentDB does not support changeStreamPreAndPostImages, even when
|
|
191
|
+
// explicitly disabled. This test only needs a collection to exercise
|
|
192
|
+
// fatal error recovery, so omit the option in DocumentDB mode.
|
|
182
193
|
changeStreamPreAndPostImages: { enabled: false }
|
|
183
194
|
});
|
|
184
195
|
const collection = db.collection('test_data');
|
|
@@ -211,7 +222,8 @@ bucket_definitions:
|
|
|
211
222
|
]);
|
|
212
223
|
});
|
|
213
224
|
|
|
214
|
-
|
|
225
|
+
// DocumentDB does not support changeStreamPreAndPostImages (post-images).
|
|
226
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('postImages - autoConfigure', async () => {
|
|
215
227
|
// Similar to the above test, but with postImages enabled.
|
|
216
228
|
// This resolves the consistency issue.
|
|
217
229
|
await using context = await openContext({
|
|
@@ -259,7 +271,8 @@ bucket_definitions:
|
|
|
259
271
|
]);
|
|
260
272
|
});
|
|
261
273
|
|
|
262
|
-
|
|
274
|
+
// DocumentDB does not support changeStreamPreAndPostImages (post-images).
|
|
275
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('postImages - on', async () => {
|
|
263
276
|
// Similar to postImages - autoConfigure, but does not auto-configure.
|
|
264
277
|
// changeStreamPreAndPostImages must be manually configured.
|
|
265
278
|
await using context = await openContext({
|
|
@@ -408,7 +421,9 @@ bucket_definitions:
|
|
|
408
421
|
]);
|
|
409
422
|
});
|
|
410
423
|
|
|
411
|
-
|
|
424
|
+
// Collection drop events are not currently replicated on DocumentDB: they are not delivered through the
|
|
425
|
+
// cluster-level change stream the way standard MongoDB delivers them, so the drop is not applied as a remove.
|
|
426
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating dropCollection', async () => {
|
|
412
427
|
await using context = await openContext();
|
|
413
428
|
const { db } = context;
|
|
414
429
|
const syncRuleContent = `
|
|
@@ -440,7 +455,9 @@ bucket_definitions:
|
|
|
440
455
|
]);
|
|
441
456
|
});
|
|
442
457
|
|
|
443
|
-
|
|
458
|
+
// Collection rename events are not currently replicated on DocumentDB: they are not delivered through the
|
|
459
|
+
// cluster-level change stream the way standard MongoDB delivers them.
|
|
460
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating renameCollection', async () => {
|
|
444
461
|
await using context = await openContext();
|
|
445
462
|
const { db } = context;
|
|
446
463
|
const syncRuleContent = `
|
|
@@ -471,7 +488,9 @@ bucket_definitions:
|
|
|
471
488
|
]);
|
|
472
489
|
});
|
|
473
490
|
|
|
474
|
-
test
|
|
491
|
+
// Concurrent-snapshot regression test driven by collection drop/recreate DDL events, which DocumentDB does
|
|
492
|
+
// not deliver through the cluster-level change stream the way standard MongoDB does.
|
|
493
|
+
test.runIf(supportsConcurrentSnapshots && DATABASE_TYPE != DatabaseType.DOCUMENTDB)(
|
|
475
494
|
'collection recreated while queued snapshot is waiting does not stall checkpoints',
|
|
476
495
|
async () => {
|
|
477
496
|
// This is a regression test for a specific timing issue in concurrent snapshot logic.
|
|
@@ -655,12 +674,14 @@ bucket_definitions:
|
|
|
655
674
|
|
|
656
675
|
// We need at least 1 commit.
|
|
657
676
|
expect(commitCount).toBeGreaterThan(0);
|
|
658
|
-
// The previous implementation
|
|
677
|
+
// The previous implementation created 1 commit per checkpoint, which is bad for performance.
|
|
659
678
|
// We expect a small number here - typically 2-10, but allow for anything less than the total number of checkpoints.
|
|
660
679
|
expect(commitCount).toBeLessThan(checkpointCount + 1);
|
|
661
680
|
});
|
|
662
681
|
|
|
663
|
-
test
|
|
682
|
+
test
|
|
683
|
+
// DocumentDBDB does not support changeStreamSplitLargeEvent
|
|
684
|
+
.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('large record', async () => {
|
|
664
685
|
// Test a large update.
|
|
665
686
|
|
|
666
687
|
// Without $changeStreamSplitLargeEvent, we get this error:
|
|
@@ -732,75 +753,83 @@ bucket_definitions:
|
|
|
732
753
|
expect(data).toMatchObject([]);
|
|
733
754
|
});
|
|
734
755
|
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
756
|
+
// DocumentDB does not support changeStreamPreAndPostImages (post-images).
|
|
757
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
|
|
758
|
+
'postImages - new collection with postImages enabled',
|
|
759
|
+
async () => {
|
|
760
|
+
await using context = await openContext({
|
|
761
|
+
mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
|
|
762
|
+
});
|
|
763
|
+
const { db } = context;
|
|
764
|
+
await context.updateSyncRules(`
|
|
741
765
|
bucket_definitions:
|
|
742
766
|
global:
|
|
743
767
|
data:
|
|
744
768
|
- SELECT _id as id, description FROM "test_%"`);
|
|
745
769
|
|
|
746
|
-
|
|
770
|
+
await context.replicateSnapshot();
|
|
747
771
|
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
772
|
+
await db.createCollection('test_data', {
|
|
773
|
+
// enabled: true here - everything should work
|
|
774
|
+
changeStreamPreAndPostImages: { enabled: true }
|
|
775
|
+
});
|
|
776
|
+
const collection = db.collection('test_data');
|
|
777
|
+
const result = await collection.insertOne({ description: 'test1' });
|
|
778
|
+
const test_id = result.insertedId;
|
|
779
|
+
await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
|
|
756
780
|
|
|
757
|
-
|
|
781
|
+
context.startStreaming();
|
|
758
782
|
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
783
|
+
const data = await context.getBucketData('global[]');
|
|
784
|
+
if (data.length == 3) {
|
|
785
|
+
expect(data).toMatchObject([
|
|
786
|
+
// An extra op here, since this triggers a snapshot in addition to getting the event.
|
|
787
|
+
test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
|
|
788
|
+
test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
|
|
789
|
+
test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
|
|
790
|
+
]);
|
|
791
|
+
} else {
|
|
792
|
+
expect(data).toMatchObject([
|
|
793
|
+
test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
|
|
794
|
+
test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
|
|
795
|
+
]);
|
|
796
|
+
}
|
|
772
797
|
}
|
|
773
|
-
|
|
798
|
+
);
|
|
774
799
|
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
800
|
+
// DocumentDB does not support changeStreamPreAndPostImages (post-images).
|
|
801
|
+
test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
|
|
802
|
+
'postImages - new collection with postImages disabled',
|
|
803
|
+
async () => {
|
|
804
|
+
await using context = await openContext({
|
|
805
|
+
mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
|
|
806
|
+
});
|
|
807
|
+
const { db } = context;
|
|
808
|
+
await context.updateSyncRules(`
|
|
781
809
|
bucket_definitions:
|
|
782
810
|
global:
|
|
783
811
|
data:
|
|
784
812
|
- SELECT _id as id, description FROM "test_data%"`);
|
|
785
813
|
|
|
786
|
-
|
|
814
|
+
await context.replicateSnapshot();
|
|
787
815
|
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
816
|
+
await db.createCollection('test_data', {
|
|
817
|
+
// enabled: false here, but autoConfigure will enable it.
|
|
818
|
+
// Unfortunately, that is too late, and replication must be restarted.
|
|
819
|
+
changeStreamPreAndPostImages: { enabled: false }
|
|
820
|
+
});
|
|
821
|
+
const collection = db.collection('test_data');
|
|
822
|
+
const result = await collection.insertOne({ description: 'test1' });
|
|
823
|
+
const test_id = result.insertedId;
|
|
824
|
+
await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
|
|
797
825
|
|
|
798
|
-
|
|
826
|
+
context.startStreaming();
|
|
799
827
|
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
828
|
+
await expect(() => context.getBucketData('global[]')).rejects.toMatchObject({
|
|
829
|
+
message: expect.stringContaining('stream was configured to require a post-image for all update events')
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
);
|
|
804
833
|
|
|
805
834
|
test('recover from error', async () => {
|
|
806
835
|
await using context = await openContext();
|
|
@@ -812,7 +841,10 @@ bucket_definitions:
|
|
|
812
841
|
- SELECT _id as id, description, num FROM "test_data"`);
|
|
813
842
|
|
|
814
843
|
await db.createCollection('test_data', {
|
|
815
|
-
|
|
844
|
+
// DocumentDB does not support changeStreamPreAndPostImages, even when
|
|
845
|
+
// explicitly disabled. This test only needs a collection to exercise
|
|
846
|
+
// fatal error recovery, so omit the option in DocumentDB mode.
|
|
847
|
+
changeStreamPreAndPostImages: DATABASE_TYPE == DatabaseType.MONGODB ? { enabled: false } : undefined
|
|
816
848
|
});
|
|
817
849
|
|
|
818
850
|
const collection = db.collection('test_data');
|
|
@@ -837,7 +869,7 @@ bucket_definitions:
|
|
|
837
869
|
const error = (await syncRules?.getSyncConfigStatus())?.last_fatal_error;
|
|
838
870
|
return error == null;
|
|
839
871
|
},
|
|
840
|
-
{ timeout: 2_000 }
|
|
872
|
+
{ timeout: testTimeout(2_000, { cloudOverride: 5_000 }) }
|
|
841
873
|
);
|
|
842
874
|
});
|
|
843
875
|
}
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
createCoreReplicationMetrics,
|
|
6
6
|
initializeCoreReplicationMetrics,
|
|
7
7
|
InternalOpId,
|
|
8
|
+
isBatchEnd,
|
|
8
9
|
LEGACY_STORAGE_VERSION,
|
|
9
10
|
OplogEntry,
|
|
10
11
|
ProtocolOpId,
|
|
@@ -19,9 +20,15 @@ import {
|
|
|
19
20
|
} from '@powersync/service-core';
|
|
20
21
|
import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
|
|
21
22
|
|
|
23
|
+
import { SentinelLSN } from '@module/common/SentinelLSN.js';
|
|
22
24
|
import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js';
|
|
23
25
|
import { MongoManager } from '@module/replication/MongoManager.js';
|
|
24
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
createCheckpoint,
|
|
28
|
+
createSentinelCheckpointLsn,
|
|
29
|
+
STANDALONE_CHECKPOINT_ID
|
|
30
|
+
} from '@module/replication/MongoRelation.js';
|
|
31
|
+
import { detectDocumentDb } from '@module/replication/replication-utils.js';
|
|
25
32
|
import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
|
|
26
33
|
|
|
27
34
|
import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
@@ -46,6 +53,11 @@ export class ChangeStreamTestContext {
|
|
|
46
53
|
storageVersion?: number;
|
|
47
54
|
mongoOptions?: Partial<NormalizedMongoConnectionConfig>;
|
|
48
55
|
streamOptions?: Partial<ChangeStreamOptions>;
|
|
56
|
+
/**
|
|
57
|
+
* Optional override for tests that need to force a mode. By default the
|
|
58
|
+
* test context detects DocumentDB from the hello response.
|
|
59
|
+
*/
|
|
60
|
+
documentDbMode?: boolean;
|
|
49
61
|
}
|
|
50
62
|
) {
|
|
51
63
|
const f = await factory({ doNotClear: options?.doNotClear });
|
|
@@ -56,15 +68,17 @@ export class ChangeStreamTestContext {
|
|
|
56
68
|
}
|
|
57
69
|
|
|
58
70
|
const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION;
|
|
71
|
+
const documentDbMode = options?.documentDbMode ?? (await detectDocumentDb(connectionManager.db));
|
|
59
72
|
|
|
60
|
-
return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion);
|
|
73
|
+
return new ChangeStreamTestContext(f, connectionManager, options?.streamOptions, storageVersion, documentDbMode);
|
|
61
74
|
}
|
|
62
75
|
|
|
63
76
|
constructor(
|
|
64
77
|
public factory: BucketStorageFactory,
|
|
65
78
|
public connectionManager: MongoManager,
|
|
66
79
|
private streamOptions: Partial<ChangeStreamOptions> = {},
|
|
67
|
-
private storageVersion: number = LEGACY_STORAGE_VERSION
|
|
80
|
+
private storageVersion: number = LEGACY_STORAGE_VERSION,
|
|
81
|
+
private documentDbMode: boolean = false
|
|
68
82
|
) {
|
|
69
83
|
createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
|
|
70
84
|
initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
|
|
@@ -120,6 +134,17 @@ export class ChangeStreamTestContext {
|
|
|
120
134
|
return this.storage!;
|
|
121
135
|
}
|
|
122
136
|
|
|
137
|
+
async loadActiveSyncRules() {
|
|
138
|
+
const syncConfig = await this.factory.getActiveSyncConfig();
|
|
139
|
+
if (syncConfig == null) {
|
|
140
|
+
throw new Error(`Active sync config not found`);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
this.syncRulesContent = syncConfig.content;
|
|
144
|
+
this.storage = syncConfig.storage;
|
|
145
|
+
return this.storage!;
|
|
146
|
+
}
|
|
147
|
+
|
|
123
148
|
private getSyncConfigContent(): storage.PersistedSyncConfigContent {
|
|
124
149
|
if (this.syncRulesContent == null) {
|
|
125
150
|
throw new Error('Sync config not configured - call updateSyncRules() first');
|
|
@@ -143,6 +168,7 @@ export class ChangeStreamTestContext {
|
|
|
143
168
|
// a long time to abort the stream.
|
|
144
169
|
maxAwaitTimeMS: this.streamOptions?.maxAwaitTimeMS ?? 200,
|
|
145
170
|
snapshotChunkLength: this.streamOptions?.snapshotChunkLength,
|
|
171
|
+
keepaliveIntervalMs: this.streamOptions?.keepaliveIntervalMs,
|
|
146
172
|
storageHooks: this.streamOptions?.storageHooks,
|
|
147
173
|
snapshotHooks: this.streamOptions?.snapshotHooks,
|
|
148
174
|
logger: this.streamOptions?.logger
|
|
@@ -170,7 +196,24 @@ export class ChangeStreamTestContext {
|
|
|
170
196
|
* would result in inconsistent data.
|
|
171
197
|
*/
|
|
172
198
|
async markSnapshotConsistent() {
|
|
173
|
-
|
|
199
|
+
let checkpoint: string;
|
|
200
|
+
if (this.documentDbMode) {
|
|
201
|
+
const sentinelCheckpoint = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(this.client, this.db));
|
|
202
|
+
const status = await this.storage!.getStatus();
|
|
203
|
+
const resumeFrom = status.resumeLsn;
|
|
204
|
+
const resumeToken = resumeFrom ? SentinelLSN.fromSerialized(resumeFrom).resumeToken : null;
|
|
205
|
+
|
|
206
|
+
// This helper artificially marks the snapshot as consistent without
|
|
207
|
+
// waiting for the stream to observe the sentinel. Keep the sentinel as the
|
|
208
|
+
// comparable position, but carry forward the existing snapshot resume
|
|
209
|
+
// token so later DocumentDB streaming still resumes from a real token.
|
|
210
|
+
checkpoint = new SentinelLSN({
|
|
211
|
+
sentinel: sentinelCheckpoint.sentinel,
|
|
212
|
+
resume_token: resumeToken
|
|
213
|
+
}).comparable;
|
|
214
|
+
} else {
|
|
215
|
+
checkpoint = await createCheckpoint(this.db, STANDALONE_CHECKPOINT_ID);
|
|
216
|
+
}
|
|
174
217
|
|
|
175
218
|
await using writer = await this.storage!.createWriter(test_utils.BATCH_OPTIONS);
|
|
176
219
|
await writer.keepalive(checkpoint);
|
|
@@ -203,23 +246,51 @@ export class ChangeStreamTestContext {
|
|
|
203
246
|
}
|
|
204
247
|
|
|
205
248
|
async getBucketData(bucket: string, start?: ProtocolOpId | InternalOpId | undefined, options?: { timeout?: number }) {
|
|
249
|
+
const checkpoint = await this.getCheckpoint(options);
|
|
250
|
+
return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async getBucketDataAtLatestCheckpoint(bucket: string, start?: ProtocolOpId | InternalOpId | undefined) {
|
|
254
|
+
if (this.storage == null) {
|
|
255
|
+
throw new Error('updateSyncRules() first');
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const checkpoint = await this.storage.getCheckpoint();
|
|
259
|
+
return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async getBucketDataAtCheckpoint(
|
|
263
|
+
bucket: string,
|
|
264
|
+
checkpoint: ReplicationCheckpoint,
|
|
265
|
+
start?: ProtocolOpId | InternalOpId | undefined
|
|
266
|
+
) {
|
|
206
267
|
start ??= 0n;
|
|
207
268
|
if (typeof start == 'string') {
|
|
208
269
|
start = BigInt(start);
|
|
209
270
|
}
|
|
210
271
|
const syncConfigContent = this.getSyncConfigContent();
|
|
211
|
-
const checkpoint = await this.getCheckpoint(options);
|
|
212
272
|
let map = [bucketRequest(syncConfigContent, bucket, start)];
|
|
213
273
|
let data: OplogEntry[] = [];
|
|
214
274
|
while (true) {
|
|
215
275
|
const batch = this.storage!.getBucketDataBatch(checkpoint, map);
|
|
216
276
|
|
|
217
|
-
const
|
|
218
|
-
|
|
219
|
-
if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
|
|
277
|
+
const chunks = await test_utils.fromAsync(batch);
|
|
278
|
+
if (chunks.length == 0) {
|
|
220
279
|
break;
|
|
221
280
|
}
|
|
222
|
-
|
|
281
|
+
for (let chunk of chunks) {
|
|
282
|
+
if (isBatchEnd(chunk)) {
|
|
283
|
+
if (!chunk.hasMore) {
|
|
284
|
+
return data;
|
|
285
|
+
}
|
|
286
|
+
} else {
|
|
287
|
+
data = data.concat(chunk.chunkData.data ?? []);
|
|
288
|
+
map = [bucketRequest(syncConfigContent, bucket, BigInt(chunk.chunkData.next_after))];
|
|
289
|
+
if (!chunk.chunkData.has_more) {
|
|
290
|
+
return data;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
223
294
|
}
|
|
224
295
|
return data;
|
|
225
296
|
}
|
|
@@ -247,14 +318,25 @@ export async function getClientCheckpoint(
|
|
|
247
318
|
client: mongo.MongoClient,
|
|
248
319
|
db: mongo.Db,
|
|
249
320
|
storageFactory: BucketStorageFactory,
|
|
250
|
-
options?: { timeout?: number }
|
|
321
|
+
options?: { timeout?: number; documentDbMode?: boolean; storage?: SyncRulesBucketStorage }
|
|
251
322
|
): Promise<ReplicationCheckpoint> {
|
|
252
323
|
const start = Date.now();
|
|
253
|
-
const
|
|
324
|
+
const documentDbMode = options?.documentDbMode ?? (await detectDocumentDb(db));
|
|
325
|
+
|
|
326
|
+
const lsn = documentDbMode
|
|
327
|
+
? await createSentinelCheckpointLsn(client, db)
|
|
328
|
+
: await createCheckpoint(db, STANDALONE_CHECKPOINT_ID);
|
|
254
329
|
// This old API needs a persisted checkpoint id.
|
|
255
330
|
// Since we don't use LSNs anymore, the only way to get that is to wait.
|
|
256
331
|
|
|
257
332
|
const timeout = options?.timeout ?? 50_000;
|
|
333
|
+
// DocumentDB: the streaming loop skips standalone checkpoint events while a
|
|
334
|
+
// batch barrier is pending (see ChangeStream.ts), so a single sentinel bump
|
|
335
|
+
// can be missed on an idle stream with no later event to advance the
|
|
336
|
+
// checkpoint. Periodically re-bump the sentinel so a standalone event
|
|
337
|
+
// eventually commits past `lsn`, mirroring getSnapshotLsn's retry loop.
|
|
338
|
+
const NUDGE_INTERVAL_MS = 1000;
|
|
339
|
+
let lastNudge = Date.now();
|
|
258
340
|
let lastCp: ReplicationCheckpoint | null = null;
|
|
259
341
|
|
|
260
342
|
while (Date.now() - start < timeout) {
|
|
@@ -266,6 +348,12 @@ export async function getClientCheckpoint(
|
|
|
266
348
|
return cp;
|
|
267
349
|
}
|
|
268
350
|
}
|
|
351
|
+
|
|
352
|
+
if (documentDbMode && Date.now() - lastNudge >= NUDGE_INTERVAL_MS) {
|
|
353
|
+
await createSentinelCheckpointLsn(client, db);
|
|
354
|
+
lastNudge = Date.now();
|
|
355
|
+
}
|
|
356
|
+
|
|
269
357
|
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
270
358
|
}
|
|
271
359
|
|
|
@@ -4,9 +4,12 @@ import { mongo } from '@powersync/lib-service-mongodb';
|
|
|
4
4
|
|
|
5
5
|
import { MongoLSN } from '@module/common/MongoLSN.js';
|
|
6
6
|
import { createCheckpoint } from '@module/replication/MongoRelation.js';
|
|
7
|
+
import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
|
|
7
8
|
import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
|
|
8
9
|
|
|
9
|
-
|
|
10
|
+
// This suite uses configureFailPoint to force retry timing. Azure DocumentDB
|
|
11
|
+
// does not support configureFailPoint.
|
|
12
|
+
describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('checkpoint retryable writes', () => {
|
|
10
13
|
test('returns the persisted checkpoint clusterTime after a retryable write', { repeats: 1 }, async (ctx) => {
|
|
11
14
|
// This test relies on very specific timing:
|
|
12
15
|
//
|
|
@@ -75,7 +78,7 @@ describe('checkpoint retryable writes', () => {
|
|
|
75
78
|
blockTimeMS: TIMEOUT
|
|
76
79
|
}
|
|
77
80
|
});
|
|
78
|
-
const returnedLsnPromise = createCheckpoint(
|
|
81
|
+
const returnedLsnPromise = createCheckpoint(db, checkpointId);
|
|
79
82
|
|
|
80
83
|
for (let i = 0; i < INSERT_COUNT; i++) {
|
|
81
84
|
await advanceClusterTime.insertOne({ at: new Date() });
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { normalizeConnectionConfig } from '@module/types/types.js';
|
|
2
|
+
import { describe, expect, test } from 'vitest';
|
|
3
|
+
|
|
4
|
+
const BASE_CONFIG = {
|
|
5
|
+
type: 'mongodb' as const,
|
|
6
|
+
uri: 'mongodb://localhost:27017/powersync_test'
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
describe('MongoDB connection config', () => {
|
|
10
|
+
test('defaults heartbeat_interval_seconds to 60 seconds', () => {
|
|
11
|
+
expect(normalizeConnectionConfig(BASE_CONFIG).heartbeat_interval_seconds).toBe(60);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test('defaults a null heartbeat_interval_seconds to 60 seconds', () => {
|
|
15
|
+
expect(
|
|
16
|
+
normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: null }).heartbeat_interval_seconds
|
|
17
|
+
).toBe(60);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test('allows the MongoDB maximum heartbeat interval', () => {
|
|
21
|
+
expect(
|
|
22
|
+
normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds: 60 }).heartbeat_interval_seconds
|
|
23
|
+
).toBe(60);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test.each([0, 4, 61, Number.NaN, Number.POSITIVE_INFINITY])(
|
|
27
|
+
'rejects invalid heartbeat_interval_seconds: %s',
|
|
28
|
+
(heartbeat_interval_seconds) => {
|
|
29
|
+
expect(() => normalizeConnectionConfig({ ...BASE_CONFIG, heartbeat_interval_seconds })).toThrow(
|
|
30
|
+
'heartbeat_interval_seconds'
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
);
|
|
34
|
+
});
|