@powersync/service-module-mongodb 0.0.0-dev-20260909133214 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +38 -6
  2. package/ci/test-connection.yaml +12 -0
  3. package/dist/api/MongoRouteAPIAdapter.d.ts +2 -2
  4. package/dist/api/MongoRouteAPIAdapter.js +73 -143
  5. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  6. package/dist/api/infer-collection-schema.d.ts +12 -0
  7. package/dist/api/infer-collection-schema.js +142 -0
  8. package/dist/api/infer-collection-schema.js.map +1 -0
  9. package/dist/index.d.ts +1 -0
  10. package/dist/index.js +1 -0
  11. package/dist/index.js.map +1 -1
  12. package/dist/module/MongoModule.d.ts +1 -2
  13. package/dist/module/MongoModule.js +0 -1
  14. package/dist/module/MongoModule.js.map +1 -1
  15. package/dist/replication/ChangeStream.d.ts +10 -2
  16. package/dist/replication/ChangeStream.js +228 -317
  17. package/dist/replication/ChangeStream.js.map +1 -1
  18. package/dist/replication/ChangeStreamReplicationJob.d.ts +4 -1
  19. package/dist/replication/ChangeStreamReplicationJob.js +4 -1
  20. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  21. package/dist/replication/ChangeStreamReplicator.d.ts +3 -0
  22. package/dist/replication/ChangeStreamReplicator.js +3 -0
  23. package/dist/replication/ChangeStreamReplicator.js.map +1 -1
  24. package/dist/replication/MongoRelation.js +5 -2
  25. package/dist/replication/MongoRelation.js.map +1 -1
  26. package/dist/replication/MongoReplicationQueryProvider.d.ts +87 -0
  27. package/dist/replication/MongoReplicationQueryProvider.js +23 -0
  28. package/dist/replication/MongoReplicationQueryProvider.js.map +1 -0
  29. package/dist/replication/MongoReplicationStream.d.ts +85 -0
  30. package/dist/replication/MongoReplicationStream.js +101 -0
  31. package/dist/replication/MongoReplicationStream.js.map +1 -0
  32. package/dist/replication/MongoSnapshotQuery.d.ts +2 -0
  33. package/dist/replication/MongoSnapshotQuery.js +12 -2
  34. package/dist/replication/MongoSnapshotQuery.js.map +1 -1
  35. package/dist/replication/MongoSnapshotter.d.ts +7 -2
  36. package/dist/replication/MongoSnapshotter.js +58 -85
  37. package/dist/replication/MongoSnapshotter.js.map +1 -1
  38. package/dist/replication/RawChangeStream.d.ts +1 -1
  39. package/dist/replication/RawChangeStream.js +1 -1
  40. package/dist/replication/RawChangeStream.js.map +1 -1
  41. package/dist/replication/SourceRowConverter.d.ts +1 -0
  42. package/dist/replication/SourceRowConverter.js +4 -2
  43. package/dist/replication/SourceRowConverter.js.map +1 -1
  44. package/dist/replication/bufferToSqlite.d.ts +3 -1
  45. package/dist/replication/bufferToSqlite.js +35 -23
  46. package/dist/replication/bufferToSqlite.js.map +1 -1
  47. package/dist/replication/replication-index.d.ts +6 -0
  48. package/dist/replication/replication-index.js +6 -0
  49. package/dist/replication/replication-index.js.map +1 -1
  50. package/dist/test-utils/ChangeStreamTestContext.d.ts +131 -0
  51. package/dist/test-utils/ChangeStreamTestContext.js +391 -0
  52. package/dist/test-utils/ChangeStreamTestContext.js.map +1 -0
  53. package/dist/test-utils/test-utils-index.d.ts +1 -0
  54. package/dist/test-utils/test-utils-index.js +2 -0
  55. package/dist/test-utils/test-utils-index.js.map +1 -0
  56. package/package.json +8 -8
  57. package/src/api/MongoRouteAPIAdapter.ts +79 -127
  58. package/src/api/infer-collection-schema.ts +153 -0
  59. package/src/index.ts +1 -0
  60. package/src/module/MongoModule.ts +1 -3
  61. package/src/replication/ChangeStream.ts +253 -359
  62. package/src/replication/ChangeStreamReplicationJob.ts +7 -2
  63. package/src/replication/ChangeStreamReplicator.ts +5 -0
  64. package/src/replication/MongoRelation.ts +8 -1
  65. package/src/replication/MongoReplicationQueryProvider.ts +108 -0
  66. package/src/replication/MongoReplicationStream.ts +201 -0
  67. package/src/replication/MongoSnapshotQuery.ts +19 -3
  68. package/src/replication/MongoSnapshotter.ts +70 -92
  69. package/src/replication/RawChangeStream.ts +2 -2
  70. package/src/replication/SourceRowConverter.ts +4 -2
  71. package/src/replication/bufferToSqlite.ts +43 -23
  72. package/src/replication/replication-index.ts +6 -0
  73. package/src/test-utils/ChangeStreamTestContext.ts +520 -0
  74. package/src/test-utils/test-utils-index.ts +1 -0
  75. package/test/src/buffer_to_sqlite.test.ts +9 -0
  76. package/test/src/change_stream.test.ts +3 -3
  77. package/test/src/change_stream_test_setup.ts +38 -0
  78. package/test/src/chunked_snapshot.test.ts +3 -3
  79. package/test/src/documentdb_mode.test.ts +3 -3
  80. package/test/src/replication_stream.test.ts +311 -0
  81. package/test/src/resume.test.ts +10 -4
  82. package/test/src/resuming_snapshots.test.ts +3 -3
  83. package/test/src/schema.test.ts +223 -0
  84. package/test/src/slow_tests.test.ts +3 -3
  85. package/test/src/snapshot_query_provider.test.ts +95 -0
  86. package/test/src/stream_progress.test.ts +312 -0
  87. package/tsconfig.tsbuildinfo +1 -1
  88. package/test/src/change_stream_utils.ts +0 -375
@@ -18,8 +18,13 @@ import { CheckpointImplementation } from './checkpoints/CheckpointImplementation
18
18
  import { createCheckpointImplementation } from './checkpoints/create-checkpoint-implementation.js';
19
19
  import { MongoManager } from './MongoManager.js';
20
20
  import { getMongoRelation } from './MongoRelation.js';
21
+ import {
22
+ DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
23
+ MongoReplicationQueryProvider
24
+ } from './MongoReplicationQueryProvider.js';
25
+ import { MongoReplicationStreamItem, openMongoReplicationStream } from './MongoReplicationStream.js';
21
26
  import { ChunkedSnapshotQuery } from './MongoSnapshotQuery.js';
22
- import { ChangeStreamBatch, parseChangeDocument, rawChangeStream } from './RawChangeStream.js';
27
+ import { ChangeStreamBatch } from './RawChangeStream.js';
23
28
  import { CHECKPOINTS_COLLECTION, detectDocumentDb } from './replication-utils.js';
24
29
  import { DirectSourceRowConverter, SourceRowConverter } from './SourceRowConverter.js';
25
30
 
@@ -34,6 +39,7 @@ export interface MongoSnapshotterOptions {
34
39
  checkpointStreamId: mongo.ObjectId;
35
40
  storageHooks?: storage.StorageHooks;
36
41
  snapshotHooks?: MongoSnapshotterHooks;
42
+ queryProvider?: MongoReplicationQueryProvider;
37
43
  }
38
44
 
39
45
  export interface MongoSnapshotterHooks {
@@ -59,6 +65,7 @@ export class MongoSnapshotter {
59
65
  private readonly defaultDb: mongo.Db;
60
66
  private readonly syncRules: HydratedSyncConfig;
61
67
  private readonly sourceRowConverter: SourceRowConverter;
68
+ private readonly queryProvider: MongoReplicationQueryProvider;
62
69
  private readonly maxAwaitTimeMS: number;
63
70
  private readonly snapshotChunkLength: number;
64
71
  private readonly abortSignal: AbortSignal;
@@ -90,6 +97,7 @@ export class MongoSnapshotter {
90
97
  this.checkpointStreamId = options.checkpointStreamId;
91
98
  this.storageHooks = options.storageHooks;
92
99
  this.snapshotHooks = options.snapshotHooks;
100
+ this.queryProvider = options.queryProvider ?? DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER;
93
101
  this.changeStreamTimeout = Math.ceil(this.client.options.socketTimeoutMS * 0.9);
94
102
  this.syncRules = options.storage.getParsedSyncRules({
95
103
  defaultSchema: this.defaultDb.databaseName
@@ -116,7 +124,9 @@ export class MongoSnapshotter {
116
124
  return this.storage.storageConfig.softDeleteCurrentData;
117
125
  }
118
126
 
119
- /** The active checkpoint strategy. Only valid after ensureDetected(). */
127
+ /**
128
+ * The active checkpoint strategy. Only valid after ensureDetected().
129
+ */
120
130
  private get checkpointImplementation(): CheckpointImplementation {
121
131
  if (this._checkpointImplementation == null) {
122
132
  throw new ServiceError(
@@ -150,6 +160,7 @@ export class MongoSnapshotter {
150
160
  );
151
161
  }
152
162
  }
163
+ await this.queryProvider.validateSource?.({ connectionManager: this.connections, isDocumentDb: this.isDocumentDb });
153
164
  this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
154
165
  client: this.client,
155
166
  db: this.defaultDb,
@@ -244,9 +255,7 @@ export class MongoSnapshotter {
244
255
  totalEstimatedCount: count
245
256
  });
246
257
  this.queueTable(updated);
247
- this.logger.info(
248
- `To replicate: ${updated.qualifiedName}: ${updated.snapshotStatus?.replicatedCount}/~${updated.snapshotStatus?.totalEstimatedCount}`
249
- );
258
+ this.logger.info(`To replicate: ${updated.qualifiedName} ${updated.formatSnapshotProgress()}`);
250
259
  }
251
260
  }
252
261
 
@@ -465,13 +474,15 @@ export class MongoSnapshotter {
465
474
  const bytesReplicatedMetric = this.metrics.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES);
466
475
  const chunksReplicatedMetric = this.metrics.getCounter(ReplicationMetric.CHUNKS_REPLICATED);
467
476
 
468
- const totalEstimatedCount = await this.estimatedCountNumber(table);
477
+ const filter = this.queryProvider.getSnapshotFilter(table);
478
+ const totalEstimatedCount = await this.estimatedCountNumber(table, filter);
469
479
  let at = table.snapshotStatus?.replicatedCount ?? 0;
470
480
  const collection = this.client.db(table.schema).collection(table.name);
471
481
  await using query = new ChunkedSnapshotQuery({
472
482
  collection,
473
483
  key: table.snapshotStatus?.lastKey,
474
- batchSize: this.snapshotChunkLength
484
+ batchSize: this.snapshotChunkLength,
485
+ filter
475
486
  });
476
487
  if (query.lastKey != null) {
477
488
  this.logger.info(
@@ -565,7 +576,15 @@ export class MongoSnapshotter {
565
576
  return result.tables;
566
577
  }
567
578
 
568
- private async estimatedCountNumber(table: storage.SourceTable): Promise<number> {
579
+ private async estimatedCountNumber(
580
+ table: storage.SourceTable,
581
+ filter = this.queryProvider.getSnapshotFilter(table)
582
+ ): Promise<number> {
583
+ // Collection metadata cannot estimate a filtered result. An exact count could require another
584
+ // expensive scan, so use the shared unknown-total sentinel and report only the replicated count.
585
+ if (filter != null) {
586
+ return -1;
587
+ }
569
588
  return await this.client.db(table.schema).collection(table.name).estimatedDocumentCount();
570
589
  }
571
590
 
@@ -616,7 +635,7 @@ export class MongoSnapshotter {
616
635
  this.checkpointImplementation.seedPosition(null);
617
636
  const startStreamFromLsn = await this.checkpointImplementation.createFirstBarrier();
618
637
  const filters = this.getSourceNamespaceFilters();
619
- const iter = this.rawChangeStreamBatches({
638
+ const iter = this.openChangeStream({
620
639
  lsn: startStreamFromLsn,
621
640
  maxAwaitTimeMS: 0,
622
641
  signal: this.abortSignal,
@@ -627,7 +646,7 @@ export class MongoSnapshotter {
627
646
  let eventsSeen = 0;
628
647
  let batchesSeen = 0;
629
648
 
630
- for await (const { events } of iter) {
649
+ for await (const item of iter) {
631
650
  if (performance.now() - startTime >= LSN_TIMEOUT_SECONDS * 1000) {
632
651
  break;
633
652
  }
@@ -635,26 +654,26 @@ export class MongoSnapshotter {
635
654
  await this.checkpointImplementation.createBatchCheckpoint();
636
655
  lastCheckpointCreated = performance.now();
637
656
  }
638
- batchesSeen += 1;
639
-
640
- for (const rawChangeDocument of events) {
641
- const changeDocument = parseChangeDocument(rawChangeDocument);
642
- const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
643
-
644
- if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
645
- const kind = this.checkpointImplementation.event.observe(changeDocument);
646
- if (kind != 'own-barrier') {
647
- // Standalone events still feed the implementation's coordinate via
648
- // event.observe above; we only resolve on our own barrier.
649
- continue;
650
- }
651
- // Observing our own barrier has set the coordinate (the barrier event
652
- // carries the sentinel counter directly), so the LSN is ready.
653
- return this.checkpointImplementation.event.lsn(changeDocument);
657
+ if (item.type == 'progress') {
658
+ batchesSeen += 1;
659
+ continue;
660
+ }
661
+ const changeDocument = item.event;
662
+ const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
663
+
664
+ if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
665
+ const kind = this.checkpointImplementation.event.observe(changeDocument);
666
+ if (kind != 'own-barrier') {
667
+ // Standalone events still feed the implementation's coordinate via
668
+ // event.observe above; we only resolve on our own barrier.
669
+ continue;
654
670
  }
655
-
656
- eventsSeen += 1;
671
+ // Observing our own barrier has set the coordinate (the barrier event
672
+ // carries the sentinel counter directly), so the LSN is ready.
673
+ return this.checkpointImplementation.event.lsn(changeDocument);
657
674
  }
675
+
676
+ eventsSeen += 1;
658
677
  }
659
678
 
660
679
  // Could happen if there is a very large replication lag?
@@ -668,12 +687,13 @@ export class MongoSnapshotter {
668
687
  * Given a snapshot LSN, validate that we can read from it, by opening a change stream.
669
688
  */
670
689
  private async validateSnapshotLsn(lsn: string) {
671
- const stream = this.rawChangeStreamBatches({
690
+ const stream = this.openChangeStream({
672
691
  lsn,
673
692
  maxAwaitTimeMS: 0,
674
- filters: this.getSourceNamespaceFilters()
693
+ filters: this.getSourceNamespaceFilters(),
694
+ signal: this.abortSignal
675
695
  });
676
- for await (const _batch of stream) {
696
+ for await (const _item of stream) {
677
697
  break;
678
698
  }
679
699
  }
@@ -722,73 +742,31 @@ export class MongoSnapshotter {
722
742
  return { $match: nsFilter, multipleDatabases };
723
743
  }
724
744
 
725
- private rawChangeStreamBatches(options: {
745
+ private openChangeStream(options: {
726
746
  lsn: string | null;
727
747
  maxAwaitTimeMS?: number;
728
748
  batchSize?: number;
729
749
  filters: { $match: any; multipleDatabases: boolean };
730
750
  signal?: AbortSignal;
731
751
  tracer?: PerformanceTracer<'changestream'>;
732
- }): AsyncIterableIterator<ChangeStreamBatch> {
733
- const position = options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null;
734
- const startAfter = position?.startAfter ?? undefined;
735
- const resumeAfter = position?.resumeAfter ?? undefined;
736
-
737
- let fullDocument: 'required' | 'updateLookup';
738
- if (this.isDocumentDb) {
739
- // DocumentDB does not support changeStreamPreAndPostImages, so 'required' won't work.
740
- fullDocument = 'updateLookup';
741
- } else if (this.usePostImages) {
742
- // 'read_only' or 'auto_configure'
743
- // Configuration happens during snapshot, or when we see new
744
- // collections.
745
- fullDocument = 'required';
746
- } else {
747
- fullDocument = 'updateLookup';
748
- }
749
- const streamOptions: mongo.ChangeStreamOptions & mongo.Document = {
750
- fullDocument
751
- };
752
- if (!this.isDocumentDb) {
753
- // DocumentDB does not support showExpandedEvents.
754
- streamOptions.showExpandedEvents = true;
755
- }
756
- const pipeline: mongo.Document[] = [{ $changeStream: streamOptions }, { $match: options.filters.$match }];
757
- if (!this.isDocumentDb) {
758
- // DocumentDB does not support $changeStreamSplitLargeEvent.
759
- pipeline.push({ $changeStreamSplitLargeEvent: {} });
760
- }
761
-
762
- // Only one of these options can be supplied at a time.
763
- if (resumeAfter) {
764
- streamOptions.resumeAfter = resumeAfter;
765
- } else if (startAfter != null) {
766
- // Legacy: We don't persist lsns without resumeTokens anymore, but we do still handle the
767
- // case if we have an old one. The sentinel implementation never produces a startAfter,
768
- // and a fresh DocumentDB stream opens from "now" with neither option set.
769
- streamOptions.startAtOperationTime = startAfter;
770
- }
771
-
772
- let watchDb: mongo.Db;
773
- if (this.isDocumentDb || options.filters.multipleDatabases) {
774
- // DocumentDB only supports cluster-level change streams.
775
- // Requires readAnyDatabase@admin on Atlas.
776
- watchDb = this.client.db('admin');
777
- streamOptions.allChangesForCluster = true;
778
- } else {
779
- // Same general result, but requires less permissions than the above
780
- watchDb = this.defaultDb;
781
- }
782
-
783
- const maxAwaitTimeMS = options.maxAwaitTimeMS ?? this.maxAwaitTimeMS;
784
-
785
- return rawChangeStream(watchDb, pipeline, {
786
- batchSize: options.batchSize ?? this.snapshotChunkLength,
787
- maxAwaitTimeMS,
788
- maxTimeMS: this.changeStreamTimeout,
789
- signal: options.signal,
790
- logger: this.logger,
791
- tracer: options.tracer
752
+ onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
753
+ }): AsyncIterableIterator<MongoReplicationStreamItem> {
754
+ return openMongoReplicationStream({
755
+ db: this.defaultDb,
756
+ queryProvider: this.queryProvider,
757
+ namespaceFilter: options.filters,
758
+ isDocumentDb: this.isDocumentDb,
759
+ usePostImages: this.usePostImages,
760
+ position: options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null,
761
+ onBatch: options.onBatch,
762
+ options: {
763
+ batchSize: options.batchSize ?? this.snapshotChunkLength,
764
+ maxAwaitTimeMS: options.maxAwaitTimeMS ?? this.maxAwaitTimeMS,
765
+ maxTimeMS: this.changeStreamTimeout,
766
+ signal: options.signal,
767
+ logger: this.logger,
768
+ tracer: options.tracer
769
+ }
792
770
  });
793
771
  }
794
772
 
@@ -364,7 +364,7 @@ type MapRawDocument<T> = T extends unknown ? { [K in keyof T]: Rawify<T[K]> } :
364
364
  export type ProjectedChangeStreamDocument =
365
365
  | Omit<mongo.ChangeStreamDropDocument, 'wallTime' | 'collectionUUID'>
366
366
  | Omit<mongo.ChangeStreamRenameDocument, 'wallTime'>
367
- | Omit<mongo.ChangeStreamDeleteDocument<Buffer>, 'wallTime'>
367
+ | Omit<mongo.ChangeStreamDeleteDocument<Buffer>, 'wallTime' | 'collectionUUID'>
368
368
  | Omit<mongo.ChangeStreamInsertDocument<Buffer>, 'wallTime'>
369
369
  | Omit<mongo.ChangeStreamUpdateDocument<Buffer>, 'wallTime' | 'updateDescription'>
370
370
  | Omit<mongo.ChangeStreamReplaceDocument<Buffer>, 'wallTime'>;
@@ -432,7 +432,7 @@ export function mapChangeStreamError(e: unknown) {
432
432
  } else if (
433
433
  isMongoServerError(e) &&
434
434
  e.codeName == 'NoMatchingDocument' &&
435
- e.errmsg?.includes('post-image was not found')
435
+ (e.errmsg?.includes('post-image was not found') || e.errmsg?.includes('pre-image was not found'))
436
436
  ) {
437
437
  throw new ChangeStreamInvalidatedError(e.errmsg, e);
438
438
  } else if (isMongoServerError(e) && e.hasErrorLabel('NonResumableChangeStreamError')) {
@@ -1,5 +1,5 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { applyRowContext, CompatibilityContext, SqliteRow } from '@powersync/service-sync-rules';
2
+ import { applyRowContext, CompatibilityContext, CompatibilityOption, SqliteRow } from '@powersync/service-sync-rules';
3
3
  import { bufferToSqlite, DateRenderMode, getDateRenderMode, parseDocumentId } from './bufferToSqlite.js';
4
4
  import { constructAfterRecord } from './MongoRelation.js';
5
5
 
@@ -46,13 +46,15 @@ export class LegacySourceRowConverter implements SourceRowConverter {
46
46
 
47
47
  export class DirectSourceRowConverter implements SourceRowConverter {
48
48
  private readonly dateRenderMode: DateRenderMode;
49
+ private readonly fixedBooleansInJson: boolean;
49
50
 
50
51
  constructor(compatibilityContext: CompatibilityContext) {
51
52
  this.dateRenderMode = getDateRenderMode(compatibilityContext);
53
+ this.fixedBooleansInJson = compatibilityContext.isEnabled(CompatibilityOption.fixedBooleanInJson);
52
54
  }
53
55
 
54
56
  rawToSqliteRow(source: Buffer): { row: SqliteRow; replicaId: any } {
55
- const row = bufferToSqlite(source, this.dateRenderMode);
57
+ const row = bufferToSqlite(source, this.dateRenderMode, this.fixedBooleansInJson);
56
58
  const replicaId = parseDocumentId(source).id;
57
59
  return { row, replicaId };
58
60
  }
@@ -80,12 +80,16 @@ const SHARED_WRITER = new JsonBufferWriter();
80
80
  *
81
81
  * @param bytes the source BSON bytes
82
82
  * @param dateRenderMode derive using getDateRenderMode(compatibilityContext)
83
+ * @param fixedBooleansInJson whether to serialize booleans in nested JSON as booleans, derived from compatibility
84
+ * context.
83
85
  *
84
86
  * @returns a SqliteRow
85
87
  */
86
- export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode): SqliteRow {
88
+ export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode, fixedBooleansInJson: boolean): SqliteRow {
87
89
  const row: SqliteRow = {};
88
90
  const jsonWriter = SHARED_WRITER;
91
+ const options: SerializeToJsonOptions = { dateRenderMode, fixedBooleans: fixedBooleansInJson };
92
+
89
93
  // BSON documents are length-prefixed and null-terminated. We parse directly
90
94
  // from raw bytes, so structural validation happens here rather than in the
91
95
  // upstream BSON decoder.
@@ -112,14 +116,14 @@ export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode): S
112
116
  }
113
117
  case BSON_TYPE_ARRAY: {
114
118
  jsonWriter.reset();
115
- const result = serializeNestedArrayToJson(bytes, offset, 0, jsonWriter, dateRenderMode);
119
+ const result = serializeNestedArrayToJson(bytes, offset, 0, jsonWriter, options);
116
120
  row[key] = jsonWriter.toString();
117
121
  offset = result.nextOffset;
118
122
  break;
119
123
  }
120
124
  case BSON_TYPE_DOCUMENT: {
121
125
  jsonWriter.reset();
122
- const result = serializeNestedObjectToJson(bytes, offset, 0, jsonWriter, dateRenderMode);
126
+ const result = serializeNestedObjectToJson(bytes, offset, 0, jsonWriter, options);
123
127
  row[key] = jsonWriter.toString();
124
128
  offset = result.nextOffset;
125
129
  break;
@@ -195,7 +199,7 @@ export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode): S
195
199
  }
196
200
  case BSON_TYPE_CODE_WITH_SCOPE: {
197
201
  jsonWriter.reset();
198
- const nextOffset = writeCodeWithScopeJson(bytes, offset, 0, jsonWriter, dateRenderMode);
202
+ const nextOffset = writeCodeWithScopeJson(bytes, offset, 0, jsonWriter, options);
199
203
  row[key] = jsonWriter.toString();
200
204
  offset = nextOffset;
201
205
  break;
@@ -437,12 +441,17 @@ function skipBsonValue(bytes: Buffer, offset: number, type: number) {
437
441
  }
438
442
  }
439
443
 
444
+ interface SerializeToJsonOptions {
445
+ dateRenderMode: DateRenderMode;
446
+ fixedBooleans: boolean;
447
+ }
448
+
440
449
  function serializeNestedObjectToJson(
441
450
  bytes: Buffer,
442
451
  offset: number,
443
452
  depth: number,
444
453
  writer: JsonBufferWriter,
445
- dateRenderMode: DateRenderMode
454
+ options: SerializeToJsonOptions
446
455
  ): { nextOffset: number } {
447
456
  if (depth > NESTED_DEPTH_LIMIT) {
448
457
  throw new Error(`json nested object depth exceeds the limit of ${NESTED_DEPTH_LIMIT}`);
@@ -475,7 +484,7 @@ function serializeNestedObjectToJson(
475
484
  type,
476
485
  depth,
477
486
  writer,
478
- dateRenderMode
487
+ options
479
488
  );
480
489
  cursor = afterValue;
481
490
  // Malformed BSON must fail fast instead of getting the parser stuck on the
@@ -499,7 +508,7 @@ function serializeNestedArrayToJson(
499
508
  offset: number,
500
509
  depth: number,
501
510
  writer: JsonBufferWriter,
502
- dateRenderMode: DateRenderMode
511
+ options: SerializeToJsonOptions
503
512
  ): { nextOffset: number } {
504
513
  if (depth > NESTED_DEPTH_LIMIT) {
505
514
  throw new Error(`json nested object depth exceeds the limit of ${NESTED_DEPTH_LIMIT}`);
@@ -527,7 +536,7 @@ function serializeNestedArrayToJson(
527
536
  type,
528
537
  depth,
529
538
  writer,
530
- dateRenderMode
539
+ options
531
540
  );
532
541
  cursor = afterValue;
533
542
  assertAdvanced(previousCursor, cursor);
@@ -547,7 +556,7 @@ function serializeNestedElementValue(
547
556
  type: number,
548
557
  depth: number,
549
558
  writer: JsonBufferWriter,
550
- dateRenderMode: DateRenderMode
559
+ options: SerializeToJsonOptions
551
560
  ): { nextOffset: number; defined: boolean } {
552
561
  switch (type) {
553
562
  case BSON_TYPE_DOUBLE: // Double
@@ -555,9 +564,9 @@ function serializeNestedElementValue(
555
564
  case BSON_TYPE_STRING: // String
556
565
  return serializeNestedStringElement(bytes, offset, writer);
557
566
  case BSON_TYPE_DOCUMENT: // Embedded document
558
- return serializeNestedObjectElement(bytes, offset, depth, writer, dateRenderMode);
567
+ return serializeNestedObjectElement(bytes, offset, depth, writer, options);
559
568
  case BSON_TYPE_ARRAY: // Array
560
- return serializeNestedArrayElement(bytes, offset, depth, writer, dateRenderMode);
569
+ return serializeNestedArrayElement(bytes, offset, depth, writer, options);
561
570
  case BSON_TYPE_BINARY: // Binary
562
571
  return serializeNestedBinaryElement(bytes, offset, writer);
563
572
  case BSON_TYPE_UNDEFINED: // Undefined
@@ -567,11 +576,22 @@ function serializeNestedElementValue(
567
576
  writer.writeQuotedHexLower(bytes, offset, 12);
568
577
  return { nextOffset: offset + 12, defined: true };
569
578
  }
570
- case BSON_TYPE_BOOLEAN: // Boolean
571
- writer.writeByte(bytes[offset] ? BYTE_ONE : BYTE_ZERO);
579
+ case BSON_TYPE_BOOLEAN: {
580
+ // Boolean
581
+ const value = !!bytes[offset];
582
+
583
+ if (options.fixedBooleans) {
584
+ const str = value ? 'true' : 'false';
585
+
586
+ writer.writeAscii(str);
587
+ } else {
588
+ writer.writeByte(value ? BYTE_ONE : BYTE_ZERO);
589
+ }
590
+
572
591
  return { nextOffset: offset + 1, defined: true };
592
+ }
573
593
  case BSON_TYPE_UTC_DATETIME: // UTC datetime
574
- return serializeNestedDateTimeElement(bytes, offset, writer, dateRenderMode);
594
+ return serializeNestedDateTimeElement(bytes, offset, writer, options.dateRenderMode);
575
595
  case BSON_TYPE_NULL: // Null
576
596
  case BSON_TYPE_MIN_KEY: // MinKey
577
597
  case BSON_TYPE_MAX_KEY: // MaxKey
@@ -586,7 +606,7 @@ function serializeNestedElementValue(
586
606
  case BSON_TYPE_SYMBOL: // Symbol
587
607
  return serializeNestedSymbolElement(bytes, offset, writer);
588
608
  case BSON_TYPE_CODE_WITH_SCOPE: // JavaScript code with scope
589
- return serializeNestedCodeWithScopeElement(bytes, offset, depth, writer, dateRenderMode);
609
+ return serializeNestedCodeWithScopeElement(bytes, offset, depth, writer, options);
590
610
  case BSON_TYPE_INT32: {
591
611
  // Int32
592
612
  writer.writeAscii(String(readInt32LE(bytes, offset)));
@@ -641,9 +661,9 @@ function serializeNestedObjectElement(
641
661
  offset: number,
642
662
  depth: number,
643
663
  writer: JsonBufferWriter,
644
- dateRenderMode: DateRenderMode
664
+ options: SerializeToJsonOptions
645
665
  ): { nextOffset: number; defined: boolean } {
646
- const result = serializeNestedObjectToJson(bytes, offset, depth + 1, writer, dateRenderMode);
666
+ const result = serializeNestedObjectToJson(bytes, offset, depth + 1, writer, options);
647
667
  return { nextOffset: result.nextOffset, defined: true };
648
668
  }
649
669
 
@@ -652,9 +672,9 @@ function serializeNestedArrayElement(
652
672
  offset: number,
653
673
  depth: number,
654
674
  writer: JsonBufferWriter,
655
- dateRenderMode: DateRenderMode
675
+ options: SerializeToJsonOptions
656
676
  ): { nextOffset: number; defined: boolean } {
657
- const result = serializeNestedArrayToJson(bytes, offset, depth + 1, writer, dateRenderMode);
677
+ const result = serializeNestedArrayToJson(bytes, offset, depth + 1, writer, options);
658
678
  return { nextOffset: result.nextOffset, defined: true };
659
679
  }
660
680
 
@@ -791,14 +811,14 @@ function writeCodeWithScopeJson(
791
811
  offset: number,
792
812
  depth: number,
793
813
  writer: JsonBufferWriter,
794
- dateRenderMode: DateRenderMode
814
+ options: SerializeToJsonOptions
795
815
  ) {
796
816
  const totalLength = readInt32LE(bytes, offset);
797
817
  const { value: code, nextOffset: afterCode } = readBsonString(bytes, offset + 4);
798
818
  writer.writeAscii('{"code":');
799
819
  writer.writeQuotedJsonString(code);
800
820
  writer.writeAscii(',"scope":');
801
- serializeNestedObjectToJson(bytes, afterCode, depth + 1, writer, dateRenderMode);
821
+ serializeNestedObjectToJson(bytes, afterCode, depth + 1, writer, options);
802
822
  writer.writeByte(BYTE_RBRACE);
803
823
  // code_w_scope carries its own total byte length, so we trust that wrapper
804
824
  // rather than reconstructing the end position from the nested scope.
@@ -885,10 +905,10 @@ function serializeNestedCodeWithScopeElement(
885
905
  offset: number,
886
906
  depth: number,
887
907
  writer: JsonBufferWriter,
888
- dateRenderMode: DateRenderMode
908
+ options: SerializeToJsonOptions
889
909
  ): { nextOffset: number; defined: boolean } {
890
910
  return {
891
- nextOffset: writeCodeWithScopeJson(bytes, offset, depth, writer, dateRenderMode),
911
+ nextOffset: writeCodeWithScopeJson(bytes, offset, depth, writer, options),
892
912
  defined: true
893
913
  };
894
914
  }
@@ -1,7 +1,13 @@
1
1
  export * from './ChangeStream.js';
2
2
  export * from './ChangeStreamReplicationJob.js';
3
3
  export * from './ChangeStreamReplicator.js';
4
+ export * from './ConnectionManagerFactory.js';
4
5
  export * from './internal-mongodb-utils.js';
6
+ export * from './MongoErrorRateLimiter.js';
5
7
  export * from './MongoRelation.js';
8
+ export * from './MongoReplicationQueryProvider.js';
9
+ export * from './MongoReplicationStream.js';
10
+ export * from './MongoSnapshotter.js';
6
11
  export * from './RawChangeStream.js';
12
+ export { CHECKPOINTS_COLLECTION } from './replication-utils.js';
7
13
  export * from './SourceRowConverter.js';