@powersync/service-module-mongodb 0.0.0-dev-20260909133214 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +38 -6
  2. package/ci/test-connection.yaml +12 -0
  3. package/dist/api/MongoRouteAPIAdapter.d.ts +2 -2
  4. package/dist/api/MongoRouteAPIAdapter.js +73 -143
  5. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  6. package/dist/api/infer-collection-schema.d.ts +12 -0
  7. package/dist/api/infer-collection-schema.js +142 -0
  8. package/dist/api/infer-collection-schema.js.map +1 -0
  9. package/dist/index.d.ts +1 -0
  10. package/dist/index.js +1 -0
  11. package/dist/index.js.map +1 -1
  12. package/dist/module/MongoModule.d.ts +1 -2
  13. package/dist/module/MongoModule.js +0 -1
  14. package/dist/module/MongoModule.js.map +1 -1
  15. package/dist/replication/ChangeStream.d.ts +10 -2
  16. package/dist/replication/ChangeStream.js +228 -317
  17. package/dist/replication/ChangeStream.js.map +1 -1
  18. package/dist/replication/ChangeStreamReplicationJob.d.ts +4 -1
  19. package/dist/replication/ChangeStreamReplicationJob.js +4 -1
  20. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  21. package/dist/replication/ChangeStreamReplicator.d.ts +3 -0
  22. package/dist/replication/ChangeStreamReplicator.js +3 -0
  23. package/dist/replication/ChangeStreamReplicator.js.map +1 -1
  24. package/dist/replication/MongoRelation.js +5 -2
  25. package/dist/replication/MongoRelation.js.map +1 -1
  26. package/dist/replication/MongoReplicationQueryProvider.d.ts +87 -0
  27. package/dist/replication/MongoReplicationQueryProvider.js +23 -0
  28. package/dist/replication/MongoReplicationQueryProvider.js.map +1 -0
  29. package/dist/replication/MongoReplicationStream.d.ts +85 -0
  30. package/dist/replication/MongoReplicationStream.js +101 -0
  31. package/dist/replication/MongoReplicationStream.js.map +1 -0
  32. package/dist/replication/MongoSnapshotQuery.d.ts +2 -0
  33. package/dist/replication/MongoSnapshotQuery.js +12 -2
  34. package/dist/replication/MongoSnapshotQuery.js.map +1 -1
  35. package/dist/replication/MongoSnapshotter.d.ts +7 -2
  36. package/dist/replication/MongoSnapshotter.js +58 -85
  37. package/dist/replication/MongoSnapshotter.js.map +1 -1
  38. package/dist/replication/RawChangeStream.d.ts +1 -1
  39. package/dist/replication/RawChangeStream.js +1 -1
  40. package/dist/replication/RawChangeStream.js.map +1 -1
  41. package/dist/replication/SourceRowConverter.d.ts +1 -0
  42. package/dist/replication/SourceRowConverter.js +4 -2
  43. package/dist/replication/SourceRowConverter.js.map +1 -1
  44. package/dist/replication/bufferToSqlite.d.ts +3 -1
  45. package/dist/replication/bufferToSqlite.js +35 -23
  46. package/dist/replication/bufferToSqlite.js.map +1 -1
  47. package/dist/replication/replication-index.d.ts +6 -0
  48. package/dist/replication/replication-index.js +6 -0
  49. package/dist/replication/replication-index.js.map +1 -1
  50. package/dist/test-utils/ChangeStreamTestContext.d.ts +131 -0
  51. package/dist/test-utils/ChangeStreamTestContext.js +391 -0
  52. package/dist/test-utils/ChangeStreamTestContext.js.map +1 -0
  53. package/dist/test-utils/test-utils-index.d.ts +1 -0
  54. package/dist/test-utils/test-utils-index.js +2 -0
  55. package/dist/test-utils/test-utils-index.js.map +1 -0
  56. package/package.json +8 -8
  57. package/src/api/MongoRouteAPIAdapter.ts +79 -127
  58. package/src/api/infer-collection-schema.ts +153 -0
  59. package/src/index.ts +1 -0
  60. package/src/module/MongoModule.ts +1 -3
  61. package/src/replication/ChangeStream.ts +253 -359
  62. package/src/replication/ChangeStreamReplicationJob.ts +7 -2
  63. package/src/replication/ChangeStreamReplicator.ts +5 -0
  64. package/src/replication/MongoRelation.ts +8 -1
  65. package/src/replication/MongoReplicationQueryProvider.ts +108 -0
  66. package/src/replication/MongoReplicationStream.ts +201 -0
  67. package/src/replication/MongoSnapshotQuery.ts +19 -3
  68. package/src/replication/MongoSnapshotter.ts +70 -92
  69. package/src/replication/RawChangeStream.ts +2 -2
  70. package/src/replication/SourceRowConverter.ts +4 -2
  71. package/src/replication/bufferToSqlite.ts +43 -23
  72. package/src/replication/replication-index.ts +6 -0
  73. package/src/test-utils/ChangeStreamTestContext.ts +520 -0
  74. package/src/test-utils/test-utils-index.ts +1 -0
  75. package/test/src/buffer_to_sqlite.test.ts +9 -0
  76. package/test/src/change_stream.test.ts +3 -3
  77. package/test/src/change_stream_test_setup.ts +38 -0
  78. package/test/src/chunked_snapshot.test.ts +3 -3
  79. package/test/src/documentdb_mode.test.ts +3 -3
  80. package/test/src/replication_stream.test.ts +311 -0
  81. package/test/src/resume.test.ts +10 -4
  82. package/test/src/resuming_snapshots.test.ts +3 -3
  83. package/test/src/schema.test.ts +223 -0
  84. package/test/src/slow_tests.test.ts +3 -3
  85. package/test/src/snapshot_query_provider.test.ts +95 -0
  86. package/test/src/stream_progress.test.ts +312 -0
  87. package/tsconfig.tsbuildinfo +1 -1
  88. package/test/src/change_stream_utils.ts +0 -375
@@ -0,0 +1,520 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { ReplicationAbortedError } from '@powersync/lib-services-framework';
3
+ import {
4
+ BucketDataRequest,
5
+ BucketStorageFactory,
6
+ createCoreReplicationMetrics,
7
+ initializeCoreReplicationMetrics,
8
+ InternalOpId,
9
+ isBatchEnd,
10
+ LEGACY_STORAGE_VERSION,
11
+ OplogEntry,
12
+ ProtocolOpId,
13
+ ReplicationCheckpoint,
14
+ settledPromise,
15
+ storage,
16
+ SyncRulesBucketStorage,
17
+ TestStorageFactory,
18
+ unsettledPromise,
19
+ updateSyncRulesFromConfig,
20
+ utils
21
+ } from '@powersync/service-core';
22
+ import { setTimeout } from 'node:timers/promises';
23
+
24
+ import { SentinelLSN } from '../common/SentinelLSN.js';
25
+ import { ChangeStream, ChangeStreamOptions } from '../replication/ChangeStream.js';
26
+ import { MongoManager } from '../replication/MongoManager.js';
27
+ import {
28
+ createCheckpoint,
29
+ createSentinelCheckpointLsn,
30
+ STANDALONE_CHECKPOINT_ID
31
+ } from '../replication/MongoRelation.js';
32
+ import { detectDocumentDb } from '../replication/replication-utils.js';
33
+ import { NormalizedMongoConnectionConfig } from '../types/types.js';
34
+
35
+ /**
36
+ * Stream settings callers may override while the context owns connections, storage, metrics and cancellation.
37
+ */
38
+ export type TestChangeStreamOptions = Omit<ChangeStreamOptions, 'connections' | 'storage' | 'metrics' | 'abort_signal'>;
39
+
40
+ export interface ChangeStreamTestContextOptions {
41
+ factory: TestStorageFactory;
42
+ connectionOptions: NormalizedMongoConnectionConfig;
43
+ metrics: ChangeStreamOptions['metrics'];
44
+ /**
45
+ * Preserve both source data and bucket storage when reopening an interrupted replication attempt.
46
+ */
47
+ doNotClear?: boolean;
48
+ storageVersion?: number;
49
+ streamOptions?: TestChangeStreamOptions;
50
+ /**
51
+ * Defaults to detecting DocumentDB from the source's hello response.
52
+ */
53
+ documentDbMode?: boolean;
54
+ }
55
+
56
+ export interface ChangeStreamTestContextResources {
57
+ factory: BucketStorageFactory;
58
+ connectionManager: MongoManager;
59
+ metrics: ChangeStreamOptions['metrics'];
60
+ storageVersion?: number;
61
+ streamOptions?: TestChangeStreamOptions;
62
+ documentDbMode?: boolean;
63
+ }
64
+
65
+ /**
66
+ * Owns source/storage connections and real replication. Callers supply their environment and storage implementation.
67
+ */
68
+ export class ChangeStreamTestContext implements AsyncDisposable {
69
+ private _walStream?: ChangeStream;
70
+ private abortController = new AbortController();
71
+ private settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
72
+ private syncRulesContent?: storage.PersistedSyncConfigContent;
73
+ private readonly replicationLeases = new Map<number, Promise<storage.ReplicationLock>>();
74
+ public storage?: SyncRulesBucketStorage;
75
+
76
+ /**
77
+ * Open a fresh fixture by default; use doNotClear to resume its persisted source and storage state.
78
+ */
79
+ static async open(options: ChangeStreamTestContextOptions): Promise<ChangeStreamTestContext> {
80
+ return this.openWith({ options, createContext: (resources) => new ChangeStreamTestContext(resources) });
81
+ }
82
+
83
+ /**
84
+ * Share resource setup and failure cleanup with subclasses that supply their own context and defaults.
85
+ */
86
+ protected static async openWith<T extends ChangeStreamTestContext>({
87
+ options,
88
+ createContext
89
+ }: {
90
+ options: ChangeStreamTestContextOptions;
91
+ createContext: (resources: ChangeStreamTestContextResources) => T | Promise<T>;
92
+ }): Promise<T> {
93
+ const factory = await options.factory({ doNotClear: options.doNotClear });
94
+ try {
95
+ const connectionManager = new MongoManager(options.connectionOptions);
96
+ try {
97
+ if (!options.doNotClear) {
98
+ await connectionManager.db.dropDatabase();
99
+ }
100
+ const documentDbMode = options.documentDbMode ?? (await detectDocumentDb(connectionManager.db));
101
+ // Await subclass setup so failures still close the shared connections below.
102
+ return await createContext({ ...options, factory, connectionManager, documentDbMode });
103
+ } catch (error) {
104
+ await connectionManager.end();
105
+ throw error;
106
+ }
107
+ } catch (error) {
108
+ await factory[Symbol.asyncDispose]();
109
+ throw error;
110
+ }
111
+ }
112
+
113
+ readonly factory: BucketStorageFactory;
114
+ readonly connectionManager: MongoManager;
115
+ protected readonly metrics: ChangeStreamOptions['metrics'];
116
+ protected readonly streamOptions: TestChangeStreamOptions;
117
+ protected readonly storageVersion: number;
118
+ private readonly documentDbMode: boolean;
119
+
120
+ /**
121
+ * Takes ownership of an already configured storage factory and source connection.
122
+ */
123
+ constructor({
124
+ factory,
125
+ connectionManager,
126
+ metrics,
127
+ streamOptions = {},
128
+ storageVersion = LEGACY_STORAGE_VERSION,
129
+ documentDbMode = false
130
+ }: ChangeStreamTestContextResources) {
131
+ this.factory = factory;
132
+ this.connectionManager = connectionManager;
133
+ this.metrics = metrics;
134
+ this.streamOptions = streamOptions;
135
+ this.storageVersion = storageVersion;
136
+ this.documentDbMode = documentDbMode;
137
+ createCoreReplicationMetrics(metrics);
138
+ initializeCoreReplicationMetrics(metrics);
139
+ const unregister = factory.registerListener({
140
+ beforeDispose: async () => {
141
+ unregister();
142
+ try {
143
+ for (const lease of this.replicationLeases.values()) {
144
+ await (await lease).release();
145
+ }
146
+ } finally {
147
+ this.replicationLeases.clear();
148
+ }
149
+ }
150
+ });
151
+ }
152
+
153
+ /**
154
+ * Abort snapshot and/or replication, without actively closing connections.
155
+ */
156
+ abort(cause?: Error) {
157
+ this.abortController.abort(cause);
158
+ }
159
+
160
+ /**
161
+ * Drain replication before inspecting its final persisted state or closing connections.
162
+ */
163
+ async stop(cause?: Error) {
164
+ this.abort(cause);
165
+ await this.settledReplicationPromise;
166
+ }
167
+
168
+ async dispose() {
169
+ await this.stop(new Error('Disposing test context'));
170
+ try {
171
+ await this.factory[Symbol.asyncDispose]();
172
+ } finally {
173
+ await this.connectionManager.end();
174
+ }
175
+ }
176
+
177
+ async [Symbol.asyncDispose]() {
178
+ await this.dispose();
179
+ }
180
+
181
+ get client() {
182
+ return this.connectionManager.client;
183
+ }
184
+
185
+ get db() {
186
+ return this.connectionManager.db;
187
+ }
188
+
189
+ get connectionTag() {
190
+ return this.connectionManager.connectionTag;
191
+ }
192
+
193
+ async updateSyncRules(content: string) {
194
+ const replicationStream = await this.factory.updateSyncRules(
195
+ // Deployment and resume must use the same parser, including caller-registered extensions.
196
+ updateSyncRulesFromConfig(
197
+ this.factory.syncConfigParser.parseContent(content, {
198
+ defaultSchema: this.db.databaseName,
199
+ throwOnError: true
200
+ }),
201
+ { storageVersion: this.storageVersion, defaultSchema: this.db.databaseName }
202
+ )
203
+ );
204
+ this.syncRulesContent = replicationStream.syncConfigContent[0];
205
+ this.storage = await this.getWritableStorage(replicationStream);
206
+ return this.storage!;
207
+ }
208
+
209
+ async loadNextSyncRules() {
210
+ const syncConfig = await this.factory.getDeployingSyncConfig();
211
+ if (syncConfig == null) {
212
+ throw new Error(`Next sync config not available`);
213
+ }
214
+
215
+ this.syncRulesContent = syncConfig.content;
216
+ this.storage = await this.getWritableStorage(syncConfig.replicationStream);
217
+ return this.storage!;
218
+ }
219
+
220
+ async loadActiveSyncRules() {
221
+ const syncConfig = await this.factory.getActiveSyncConfig();
222
+ if (syncConfig == null) {
223
+ throw new Error(`Active sync config not found`);
224
+ }
225
+
226
+ this.syncRulesContent = syncConfig.content;
227
+ this.storage = await this.getWritableStorage(syncConfig.replicationStream);
228
+ return this.storage!;
229
+ }
230
+
231
+ /**
232
+ * Reuse a replication lease per stream until the owning factory is disposed.
233
+ */
234
+ private async getWritableStorage(stream: storage.PersistedReplicationStream): Promise<SyncRulesBucketStorage> {
235
+ let pending = this.replicationLeases.get(stream.replicationStreamId);
236
+ if (pending == null) {
237
+ pending = stream.lock();
238
+ this.replicationLeases.set(stream.replicationStreamId, pending);
239
+ }
240
+ let replicationLock: storage.ReplicationLock;
241
+ try {
242
+ replicationLock = await pending;
243
+ } catch (error) {
244
+ this.replicationLeases.delete(stream.replicationStreamId);
245
+ throw error;
246
+ }
247
+ return this.factory.getInstance(stream, { replicationLock });
248
+ }
249
+
250
+ getSyncConfigContent(): storage.PersistedSyncConfigContent {
251
+ if (this.syncRulesContent == null) {
252
+ throw new Error('Sync config not configured - call updateSyncRules() first');
253
+ }
254
+ return this.syncRulesContent;
255
+ }
256
+
257
+ get streamer() {
258
+ if (this.storage == null) {
259
+ throw new Error('updateSyncRules() first');
260
+ }
261
+ if (this._walStream) {
262
+ return this._walStream;
263
+ }
264
+ const options: ChangeStreamOptions = {
265
+ // Forward all supported stream options, including custom query providers and snapshot/storage hooks.
266
+ ...this.streamOptions,
267
+ storage: this.storage,
268
+ metrics: this.metrics,
269
+ connections: this.connectionManager,
270
+ abort_signal: this.abortController.signal,
271
+ // A shorter await also keeps aborts responsive on MongoDB <= 6.0.
272
+ maxAwaitTimeMS: this.streamOptions.maxAwaitTimeMS ?? 200
273
+ };
274
+ this._walStream = new ChangeStream(options);
275
+ return this._walStream!;
276
+ }
277
+
278
+ async replicateSnapshot() {
279
+ this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
280
+ try {
281
+ await Promise.race([unsettledPromise(this.settledReplicationPromise), this.streamer.waitForInitialSnapshot()]);
282
+ } catch (e) {
283
+ if (e instanceof ReplicationAbortedError && e.cause != null) {
284
+ throw e.cause;
285
+ }
286
+ throw e;
287
+ }
288
+ }
289
+
290
+ /**
291
+ * A snapshot is not consistent until streaming replication has caught up.
292
+ * We simulate that for tests.
293
+ * Do not use if there are any writes performed while doing the snapshot, as that
294
+ * would result in inconsistent data.
295
+ */
296
+ async markSnapshotConsistent() {
297
+ let checkpoint: string;
298
+ if (this.documentDbMode) {
299
+ const sentinelCheckpoint = SentinelLSN.fromSerialized(await createSentinelCheckpointLsn(this.client, this.db));
300
+ const status = await this.storage!.getStatus();
301
+ const resumeFrom = status.resumeLsn;
302
+ const resumeToken = resumeFrom ? SentinelLSN.fromSerialized(resumeFrom).resumeToken : null;
303
+
304
+ // This helper artificially marks the snapshot as consistent without
305
+ // waiting for the stream to observe the sentinel. Keep the sentinel as the
306
+ // comparable position, but carry forward the existing snapshot resume
307
+ // token so later DocumentDB streaming still resumes from a real token.
308
+ checkpoint = new SentinelLSN({
309
+ sentinel: sentinelCheckpoint.sentinel,
310
+ resume_token: resumeToken
311
+ }).comparable;
312
+ } else {
313
+ checkpoint = await createCheckpoint(this.db, STANDALONE_CHECKPOINT_ID);
314
+ }
315
+
316
+ await using writer = await this.storage!.createWriter({
317
+ zeroLSN: '0000000000000000',
318
+ defaultSchema: this.db.databaseName,
319
+ storeCurrentData: true
320
+ });
321
+ await writer.keepalive(checkpoint);
322
+ await writer.flush();
323
+ }
324
+
325
+ startStreaming() {
326
+ this.settledReplicationPromise ??= settledPromise(this.streamer.replicate());
327
+ return this.settledReplicationPromise;
328
+ }
329
+
330
+ /**
331
+ * Wait for this stream to publish a real source marker; stop polling if replication exits first.
332
+ */
333
+ async getCheckpoint(options?: { timeout?: number }): Promise<ReplicationCheckpoint> {
334
+ if (this.settledReplicationPromise == null) {
335
+ throw new Error('Start replication before requesting a checkpoint.');
336
+ }
337
+ const polling = new AbortController();
338
+ const wait = getClientCheckpoint({
339
+ client: this.client,
340
+ db: this.db,
341
+ storageFactory: this.factory,
342
+ storage: this.storage,
343
+ timeout: options?.timeout ?? 15_000,
344
+ documentDbMode: this.documentDbMode,
345
+ signal: polling.signal
346
+ });
347
+ try {
348
+ return await Promise.race([
349
+ wait,
350
+ unsettledPromise(this.settledReplicationPromise).then(() => {
351
+ throw new Error('Replication exited before publishing the requested checkpoint.');
352
+ })
353
+ ]);
354
+ } finally {
355
+ polling.abort();
356
+ await wait.catch(() => {});
357
+ }
358
+ }
359
+
360
+ async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
361
+ let checkpoint = await this.getCheckpoint(options);
362
+ const map = Object.entries(buckets).map(([bucket, start]) => this.bucketRequest(bucket, start));
363
+ const chunks = [];
364
+ for await (const chunk of this.storage!.getBucketDataBatch(checkpoint, map)) {
365
+ chunks.push(chunk);
366
+ }
367
+ return chunks;
368
+ }
369
+
370
+ async getBucketData(bucket: string, start?: ProtocolOpId | InternalOpId | undefined, options?: { timeout?: number }) {
371
+ const checkpoint = await this.getCheckpoint(options);
372
+ return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
373
+ }
374
+
375
+ async getBucketDataAtLatestCheckpoint(bucket: string, start?: ProtocolOpId | InternalOpId | undefined) {
376
+ if (this.storage == null) {
377
+ throw new Error('updateSyncRules() first');
378
+ }
379
+
380
+ const checkpoint = await this.storage.getCheckpoint();
381
+ return this.getBucketDataAtCheckpoint(bucket, checkpoint, start);
382
+ }
383
+
384
+ async getBucketDataAtCheckpoint(
385
+ bucket: string,
386
+ checkpoint: ReplicationCheckpoint,
387
+ start?: ProtocolOpId | InternalOpId | undefined
388
+ ) {
389
+ start ??= 0n;
390
+ if (typeof start == 'string') {
391
+ start = BigInt(start);
392
+ }
393
+ let map = [this.bucketRequest(bucket, start)];
394
+ let data: OplogEntry[] = [];
395
+ while (true) {
396
+ const batch = this.storage!.getBucketDataBatch(checkpoint, map);
397
+
398
+ const chunks = [];
399
+ for await (const chunk of batch) {
400
+ chunks.push(chunk);
401
+ }
402
+ if (chunks.length == 0) {
403
+ break;
404
+ }
405
+ for (let chunk of chunks) {
406
+ if (isBatchEnd(chunk)) {
407
+ if (!chunk.hasMore) {
408
+ return data;
409
+ }
410
+ } else {
411
+ data = data.concat(chunk.chunkData.data ?? []);
412
+ map = [this.bucketRequest(bucket, BigInt(chunk.chunkData.next_after))];
413
+ if (!chunk.chunkData.has_more) {
414
+ return data;
415
+ }
416
+ }
417
+ }
418
+ }
419
+ return data;
420
+ }
421
+
422
+ /**
423
+ * Resolve a readable test bucket such as global[] to its persisted, versioned bucket name.
424
+ */
425
+ private bucketRequest(bucket: string, start: InternalOpId): BucketDataRequest {
426
+ const parsed = this.getSyncConfigContent().parsed({ defaultSchema: this.db.databaseName });
427
+ const parameterStart = bucket.indexOf('[');
428
+ const definitionName = bucket.substring(0, parameterStart);
429
+ const source = parsed.syncConfigs
430
+ .flatMap((config) => config.config.bucketDataSources)
431
+ .find((source) => source.uniqueName == definitionName);
432
+ if (source == null) {
433
+ throw new Error(`Unknown bucket '${bucket}'.`);
434
+ }
435
+ return {
436
+ bucket: parsed.hydrationState.getBucketSourceScope(source).bucketPrefix + bucket.substring(parameterStart),
437
+ source,
438
+ start
439
+ };
440
+ }
441
+
442
+ async getChecksums(buckets: string[], options?: { timeout?: number }): Promise<utils.ChecksumMap> {
443
+ let checkpoint = await this.getCheckpoint(options);
444
+ const versionedBuckets = buckets.map((bucket) => this.bucketRequest(bucket, 0n));
445
+ const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
446
+
447
+ const unversioned: utils.ChecksumMap = new Map();
448
+ for (let i = 0; i < buckets.length; i++) {
449
+ unversioned.set(buckets[i], checksums.get(versionedBuckets[i].bucket)!);
450
+ }
451
+ return unversioned;
452
+ }
453
+
454
+ async getChecksum(bucket: string, options?: { timeout?: number }) {
455
+ const checksums = await this.getChecksums([bucket], options);
456
+ return checksums.get(bucket);
457
+ }
458
+ }
459
+
460
+ export async function getClientCheckpoint({
461
+ client,
462
+ db,
463
+ storageFactory,
464
+ timeout = 50_000,
465
+ documentDbMode: forcedDocumentDbMode,
466
+ storage: expectedStorage,
467
+ signal
468
+ }: {
469
+ client: mongo.MongoClient;
470
+ db: mongo.Db;
471
+ storageFactory: BucketStorageFactory;
472
+ timeout?: number;
473
+ documentDbMode?: boolean;
474
+ /**
475
+ * Ignore checkpoints from another active stream while this one is still deploying.
476
+ */
477
+ storage?: SyncRulesBucketStorage;
478
+ signal?: AbortSignal;
479
+ }): Promise<ReplicationCheckpoint> {
480
+ const start = Date.now();
481
+ const documentDbMode = forcedDocumentDbMode ?? (await detectDocumentDb(db));
482
+
483
+ const lsn = documentDbMode
484
+ ? await createSentinelCheckpointLsn(client, db)
485
+ : await createCheckpoint(db, STANDALONE_CHECKPOINT_ID);
486
+ // The marker establishes a source position. Wait until replication publishes a client-visible checkpoint past it.
487
+
488
+ // DocumentDB: the streaming loop skips standalone checkpoint events while a
489
+ // batch barrier is pending (see ChangeStream.ts), so a single sentinel bump
490
+ // can be missed on an idle stream with no later event to advance the
491
+ // checkpoint. Periodically re-bump the sentinel so a standalone event
492
+ // eventually commits past `lsn`, mirroring getSnapshotLsn's retry loop.
493
+ const NUDGE_INTERVAL_MS = 1000;
494
+ let lastNudge = Date.now();
495
+ let lastCp: ReplicationCheckpoint | null = null;
496
+
497
+ while (Date.now() - start < timeout) {
498
+ signal?.throwIfAborted();
499
+ const activeStorage = (await storageFactory.getActiveSyncConfig())?.storage;
500
+ const cp =
501
+ expectedStorage == null || activeStorage?.replicationStreamId == expectedStorage.replicationStreamId
502
+ ? await activeStorage?.getCheckpoint()
503
+ : undefined;
504
+ if (cp != null) {
505
+ lastCp = cp;
506
+ if (cp.lsn && cp.lsn >= lsn) {
507
+ return cp;
508
+ }
509
+ }
510
+
511
+ if (documentDbMode && Date.now() - lastNudge >= NUDGE_INTERVAL_MS) {
512
+ await createSentinelCheckpointLsn(client, db);
513
+ lastNudge = Date.now();
514
+ }
515
+
516
+ await setTimeout(30, undefined, { signal });
517
+ }
518
+
519
+ throw new Error(`Timeout while waiting for checkpoint ${lsn}. Last checkpoint: ${lastCp?.lsn}`);
520
+ }
@@ -0,0 +1 @@
1
+ export * from './ChangeStreamTestContext.js';
@@ -121,6 +121,15 @@ const testCases: ConverterCase[] = [
121
121
  serializableCase('array', [1, 'two', false, null, { deep: 3 }], jsonTextPlacements('[1,"two",0,null,{"deep":3}]')),
122
122
  serializableCase('objectId', objectId, jsonStringPlacements('66e834cc91d805df11fa0ecb')),
123
123
  serializableCase('bool', true, placements(1n, '[1]', '{"nested":1}')),
124
+ {
125
+ name: 'bool (fixed json)',
126
+ buildBuffer: (placement) => serializeCaseDocument(`bool (fixed json):${placement}`, placement, false),
127
+ expected: placements(0n, '[false]', '{"nested":false}'),
128
+ context: new CompatibilityContext({
129
+ edition: CompatibilityEdition.COMPILED_STREAMS,
130
+ overrides: new Map([[CompatibilityOption.fixedBooleanInJson, true]])
131
+ })
132
+ } satisfies ConverterCase,
124
133
  serializableCase('date', normalDate, jsonStringPlacements('2023-03-06 13:47:00.000Z')),
125
134
  serializableCase('date:+010000', positiveExtendedDate, jsonStringPlacements('+010000-01-01 00:00:00.000Z')),
126
135
  serializableCase('date:-000001', negativeExtendedDate, jsonStringPlacements('-000001-12-31 23:59:59.999Z')),
@@ -8,7 +8,7 @@ import { test_utils } from '@powersync/service-core-tests';
8
8
 
9
9
  import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
10
10
  import { PostImagesOption } from '@module/types/types.js';
11
- import { ChangeStreamTestContext } from './change_stream_utils.js';
11
+ import { openChangeStreamTestContext } from './change_stream_test_setup.js';
12
12
  import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
13
13
  import { testTimeout } from './test-timeouts.js';
14
14
  import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
@@ -27,8 +27,8 @@ describe('change stream', () => {
27
27
  function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
28
28
  const supportsConcurrentSnapshots = storageVersion >= 3;
29
29
 
30
- const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
31
- return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
30
+ const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
31
+ return openChangeStreamTestContext(factory, { ...options, storageVersion });
32
32
  };
33
33
  // DocumentDB does not support changeStreamPreAndPostImages, which this test requires via postImages: READ_ONLY.
34
34
  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating basic values', async () => {
@@ -0,0 +1,38 @@
1
+ import { test_utils } from '@module/index.js';
2
+ import { mongo } from '@powersync/lib-service-mongodb';
3
+ import { TestStorageFactory } from '@powersync/service-core';
4
+ import { METRICS_HELPER } from '@powersync/service-core-tests';
5
+ import { TEST_CONNECTION_OPTIONS } from './util.js';
6
+
7
+ export const ChangeStreamTestContext = test_utils.ChangeStreamTestContext;
8
+
9
+ /**
10
+ * Supply this suite's environment and metrics to the exported context.
11
+ */
12
+ export function openChangeStreamTestContext(
13
+ factory: TestStorageFactory,
14
+ options?: Omit<test_utils.ChangeStreamTestContextOptions, 'factory' | 'connectionOptions' | 'metrics'> & {
15
+ mongoOptions?: Partial<test_utils.ChangeStreamTestContextOptions['connectionOptions']>;
16
+ }
17
+ ) {
18
+ return ChangeStreamTestContext.open({
19
+ ...options,
20
+ factory,
21
+ connectionOptions: { ...TEST_CONNECTION_OPTIONS, ...options?.mongoOptions },
22
+ metrics: METRICS_HELPER.metricsEngine
23
+ });
24
+ }
25
+
26
+ export async function setSnapshotHistorySeconds(client: mongo.MongoClient, seconds: number) {
27
+ const { minSnapshotHistoryWindowInSeconds: currentValue } = await client
28
+ .db('admin')
29
+ .command({ getParameter: 1, minSnapshotHistoryWindowInSeconds: 1 });
30
+
31
+ await client.db('admin').command({ setParameter: 1, minSnapshotHistoryWindowInSeconds: seconds });
32
+
33
+ return {
34
+ async [Symbol.asyncDispose]() {
35
+ await client.db('admin').command({ setParameter: 1, minSnapshotHistoryWindowInSeconds: currentValue });
36
+ }
37
+ };
38
+ }
@@ -5,7 +5,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
5
5
  import { SqliteJsonValue } from '@powersync/service-sync-rules';
6
6
  import * as timers from 'timers/promises';
7
7
  import { describe, expect, test } from 'vitest';
8
- import { ChangeStreamTestContext } from './change_stream_utils.js';
8
+ import { openChangeStreamTestContext } from './change_stream_test_setup.js';
9
9
  import { describeWithStorage, StorageVersionTestContext } from './util.js';
10
10
 
11
11
  describe('chunked snapshots', () => {
@@ -13,8 +13,8 @@ describe('chunked snapshots', () => {
13
13
  });
14
14
 
15
15
  function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
16
- const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
17
- return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
16
+ const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
17
+ return openChangeStreamTestContext(factory, { ...options, storageVersion });
18
18
  };
19
19
 
20
20
  // This is not as sensitive to the id type as postgres, but we still test a couple of cases
@@ -9,7 +9,7 @@ import { SentinelLSN } from '@module/common/SentinelLSN.js';
9
9
  import { createSentinelCheckpointLsn, SENTINEL_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
10
10
  import { CHECKPOINTS_COLLECTION } from '@module/replication/replication-utils.js';
11
11
  import { mongo } from '@powersync/lib-service-mongodb';
12
- import { ChangeStreamTestContext } from './change_stream_utils.js';
12
+ import { openChangeStreamTestContext } from './change_stream_test_setup.js';
13
13
  import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
14
14
  import { env } from './env.js';
15
15
  import { testTimeout } from './test-timeouts.js';
@@ -584,8 +584,8 @@ describe.skipIf(DATABASE_TYPE != DatabaseType.DOCUMENTDB)('documentDbMode', () =
584
584
  });
585
585
 
586
586
  function defineDocumentDBDbModeTests({ factory, storageVersion }: StorageVersionTestContext) {
587
- const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
588
- return ChangeStreamTestContext.open(factory, {
587
+ const openContext = (options?: Parameters<typeof openChangeStreamTestContext>[1]) => {
588
+ return openChangeStreamTestContext(factory, {
589
589
  ...options,
590
590
  storageVersion,
591
591
  streamOptions: {