@powersync/service-module-mongodb 0.0.0-dev-20260909133214 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -6
- package/ci/test-connection.yaml +12 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +2 -2
- package/dist/api/MongoRouteAPIAdapter.js +73 -143
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/api/infer-collection-schema.d.ts +12 -0
- package/dist/api/infer-collection-schema.js +142 -0
- package/dist/api/infer-collection-schema.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/module/MongoModule.d.ts +1 -2
- package/dist/module/MongoModule.js +0 -1
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +10 -2
- package/dist/replication/ChangeStream.js +228 -317
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.d.ts +4 -1
- package/dist/replication/ChangeStreamReplicationJob.js +4 -1
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/ChangeStreamReplicator.d.ts +3 -0
- package/dist/replication/ChangeStreamReplicator.js +3 -0
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/dist/replication/MongoRelation.js +5 -2
- package/dist/replication/MongoRelation.js.map +1 -1
- package/dist/replication/MongoReplicationQueryProvider.d.ts +87 -0
- package/dist/replication/MongoReplicationQueryProvider.js +23 -0
- package/dist/replication/MongoReplicationQueryProvider.js.map +1 -0
- package/dist/replication/MongoReplicationStream.d.ts +85 -0
- package/dist/replication/MongoReplicationStream.js +101 -0
- package/dist/replication/MongoReplicationStream.js.map +1 -0
- package/dist/replication/MongoSnapshotQuery.d.ts +2 -0
- package/dist/replication/MongoSnapshotQuery.js +12 -2
- package/dist/replication/MongoSnapshotQuery.js.map +1 -1
- package/dist/replication/MongoSnapshotter.d.ts +7 -2
- package/dist/replication/MongoSnapshotter.js +58 -85
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +1 -1
- package/dist/replication/RawChangeStream.js +1 -1
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/SourceRowConverter.d.ts +1 -0
- package/dist/replication/SourceRowConverter.js +4 -2
- package/dist/replication/SourceRowConverter.js.map +1 -1
- package/dist/replication/bufferToSqlite.d.ts +3 -1
- package/dist/replication/bufferToSqlite.js +35 -23
- package/dist/replication/bufferToSqlite.js.map +1 -1
- package/dist/replication/replication-index.d.ts +6 -0
- package/dist/replication/replication-index.js +6 -0
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/test-utils/ChangeStreamTestContext.d.ts +131 -0
- package/dist/test-utils/ChangeStreamTestContext.js +391 -0
- package/dist/test-utils/ChangeStreamTestContext.js.map +1 -0
- package/dist/test-utils/test-utils-index.d.ts +1 -0
- package/dist/test-utils/test-utils-index.js +2 -0
- package/dist/test-utils/test-utils-index.js.map +1 -0
- package/package.json +8 -8
- package/src/api/MongoRouteAPIAdapter.ts +79 -127
- package/src/api/infer-collection-schema.ts +153 -0
- package/src/index.ts +1 -0
- package/src/module/MongoModule.ts +1 -3
- package/src/replication/ChangeStream.ts +253 -359
- package/src/replication/ChangeStreamReplicationJob.ts +7 -2
- package/src/replication/ChangeStreamReplicator.ts +5 -0
- package/src/replication/MongoRelation.ts +8 -1
- package/src/replication/MongoReplicationQueryProvider.ts +108 -0
- package/src/replication/MongoReplicationStream.ts +201 -0
- package/src/replication/MongoSnapshotQuery.ts +19 -3
- package/src/replication/MongoSnapshotter.ts +70 -92
- package/src/replication/RawChangeStream.ts +2 -2
- package/src/replication/SourceRowConverter.ts +4 -2
- package/src/replication/bufferToSqlite.ts +43 -23
- package/src/replication/replication-index.ts +6 -0
- package/src/test-utils/ChangeStreamTestContext.ts +520 -0
- package/src/test-utils/test-utils-index.ts +1 -0
- package/test/src/buffer_to_sqlite.test.ts +9 -0
- package/test/src/change_stream.test.ts +3 -3
- package/test/src/change_stream_test_setup.ts +38 -0
- package/test/src/chunked_snapshot.test.ts +3 -3
- package/test/src/documentdb_mode.test.ts +3 -3
- package/test/src/replication_stream.test.ts +311 -0
- package/test/src/resume.test.ts +10 -4
- package/test/src/resuming_snapshots.test.ts +3 -3
- package/test/src/schema.test.ts +223 -0
- package/test/src/slow_tests.test.ts +3 -3
- package/test/src/snapshot_query_provider.test.ts +95 -0
- package/test/src/stream_progress.test.ts +312 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/test/src/change_stream_utils.ts +0 -375
|
@@ -18,8 +18,13 @@ import { CheckpointImplementation } from './checkpoints/CheckpointImplementation
|
|
|
18
18
|
import { createCheckpointImplementation } from './checkpoints/create-checkpoint-implementation.js';
|
|
19
19
|
import { MongoManager } from './MongoManager.js';
|
|
20
20
|
import { getMongoRelation } from './MongoRelation.js';
|
|
21
|
+
import {
|
|
22
|
+
DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
|
|
23
|
+
MongoReplicationQueryProvider
|
|
24
|
+
} from './MongoReplicationQueryProvider.js';
|
|
25
|
+
import { MongoReplicationStreamItem, openMongoReplicationStream } from './MongoReplicationStream.js';
|
|
21
26
|
import { ChunkedSnapshotQuery } from './MongoSnapshotQuery.js';
|
|
22
|
-
import { ChangeStreamBatch
|
|
27
|
+
import { ChangeStreamBatch } from './RawChangeStream.js';
|
|
23
28
|
import { CHECKPOINTS_COLLECTION, detectDocumentDb } from './replication-utils.js';
|
|
24
29
|
import { DirectSourceRowConverter, SourceRowConverter } from './SourceRowConverter.js';
|
|
25
30
|
|
|
@@ -34,6 +39,7 @@ export interface MongoSnapshotterOptions {
|
|
|
34
39
|
checkpointStreamId: mongo.ObjectId;
|
|
35
40
|
storageHooks?: storage.StorageHooks;
|
|
36
41
|
snapshotHooks?: MongoSnapshotterHooks;
|
|
42
|
+
queryProvider?: MongoReplicationQueryProvider;
|
|
37
43
|
}
|
|
38
44
|
|
|
39
45
|
export interface MongoSnapshotterHooks {
|
|
@@ -59,6 +65,7 @@ export class MongoSnapshotter {
|
|
|
59
65
|
private readonly defaultDb: mongo.Db;
|
|
60
66
|
private readonly syncRules: HydratedSyncConfig;
|
|
61
67
|
private readonly sourceRowConverter: SourceRowConverter;
|
|
68
|
+
private readonly queryProvider: MongoReplicationQueryProvider;
|
|
62
69
|
private readonly maxAwaitTimeMS: number;
|
|
63
70
|
private readonly snapshotChunkLength: number;
|
|
64
71
|
private readonly abortSignal: AbortSignal;
|
|
@@ -90,6 +97,7 @@ export class MongoSnapshotter {
|
|
|
90
97
|
this.checkpointStreamId = options.checkpointStreamId;
|
|
91
98
|
this.storageHooks = options.storageHooks;
|
|
92
99
|
this.snapshotHooks = options.snapshotHooks;
|
|
100
|
+
this.queryProvider = options.queryProvider ?? DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER;
|
|
93
101
|
this.changeStreamTimeout = Math.ceil(this.client.options.socketTimeoutMS * 0.9);
|
|
94
102
|
this.syncRules = options.storage.getParsedSyncRules({
|
|
95
103
|
defaultSchema: this.defaultDb.databaseName
|
|
@@ -116,7 +124,9 @@ export class MongoSnapshotter {
|
|
|
116
124
|
return this.storage.storageConfig.softDeleteCurrentData;
|
|
117
125
|
}
|
|
118
126
|
|
|
119
|
-
/**
|
|
127
|
+
/**
|
|
128
|
+
* The active checkpoint strategy. Only valid after ensureDetected().
|
|
129
|
+
*/
|
|
120
130
|
private get checkpointImplementation(): CheckpointImplementation {
|
|
121
131
|
if (this._checkpointImplementation == null) {
|
|
122
132
|
throw new ServiceError(
|
|
@@ -150,6 +160,7 @@ export class MongoSnapshotter {
|
|
|
150
160
|
);
|
|
151
161
|
}
|
|
152
162
|
}
|
|
163
|
+
await this.queryProvider.validateSource?.({ connectionManager: this.connections, isDocumentDb: this.isDocumentDb });
|
|
153
164
|
this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
|
|
154
165
|
client: this.client,
|
|
155
166
|
db: this.defaultDb,
|
|
@@ -244,9 +255,7 @@ export class MongoSnapshotter {
|
|
|
244
255
|
totalEstimatedCount: count
|
|
245
256
|
});
|
|
246
257
|
this.queueTable(updated);
|
|
247
|
-
this.logger.info(
|
|
248
|
-
`To replicate: ${updated.qualifiedName}: ${updated.snapshotStatus?.replicatedCount}/~${updated.snapshotStatus?.totalEstimatedCount}`
|
|
249
|
-
);
|
|
258
|
+
this.logger.info(`To replicate: ${updated.qualifiedName} ${updated.formatSnapshotProgress()}`);
|
|
250
259
|
}
|
|
251
260
|
}
|
|
252
261
|
|
|
@@ -465,13 +474,15 @@ export class MongoSnapshotter {
|
|
|
465
474
|
const bytesReplicatedMetric = this.metrics.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES);
|
|
466
475
|
const chunksReplicatedMetric = this.metrics.getCounter(ReplicationMetric.CHUNKS_REPLICATED);
|
|
467
476
|
|
|
468
|
-
const
|
|
477
|
+
const filter = this.queryProvider.getSnapshotFilter(table);
|
|
478
|
+
const totalEstimatedCount = await this.estimatedCountNumber(table, filter);
|
|
469
479
|
let at = table.snapshotStatus?.replicatedCount ?? 0;
|
|
470
480
|
const collection = this.client.db(table.schema).collection(table.name);
|
|
471
481
|
await using query = new ChunkedSnapshotQuery({
|
|
472
482
|
collection,
|
|
473
483
|
key: table.snapshotStatus?.lastKey,
|
|
474
|
-
batchSize: this.snapshotChunkLength
|
|
484
|
+
batchSize: this.snapshotChunkLength,
|
|
485
|
+
filter
|
|
475
486
|
});
|
|
476
487
|
if (query.lastKey != null) {
|
|
477
488
|
this.logger.info(
|
|
@@ -565,7 +576,15 @@ export class MongoSnapshotter {
|
|
|
565
576
|
return result.tables;
|
|
566
577
|
}
|
|
567
578
|
|
|
568
|
-
private async estimatedCountNumber(
|
|
579
|
+
private async estimatedCountNumber(
|
|
580
|
+
table: storage.SourceTable,
|
|
581
|
+
filter = this.queryProvider.getSnapshotFilter(table)
|
|
582
|
+
): Promise<number> {
|
|
583
|
+
// Collection metadata cannot estimate a filtered result. An exact count could require another
|
|
584
|
+
// expensive scan, so use the shared unknown-total sentinel and report only the replicated count.
|
|
585
|
+
if (filter != null) {
|
|
586
|
+
return -1;
|
|
587
|
+
}
|
|
569
588
|
return await this.client.db(table.schema).collection(table.name).estimatedDocumentCount();
|
|
570
589
|
}
|
|
571
590
|
|
|
@@ -616,7 +635,7 @@ export class MongoSnapshotter {
|
|
|
616
635
|
this.checkpointImplementation.seedPosition(null);
|
|
617
636
|
const startStreamFromLsn = await this.checkpointImplementation.createFirstBarrier();
|
|
618
637
|
const filters = this.getSourceNamespaceFilters();
|
|
619
|
-
const iter = this.
|
|
638
|
+
const iter = this.openChangeStream({
|
|
620
639
|
lsn: startStreamFromLsn,
|
|
621
640
|
maxAwaitTimeMS: 0,
|
|
622
641
|
signal: this.abortSignal,
|
|
@@ -627,7 +646,7 @@ export class MongoSnapshotter {
|
|
|
627
646
|
let eventsSeen = 0;
|
|
628
647
|
let batchesSeen = 0;
|
|
629
648
|
|
|
630
|
-
for await (const
|
|
649
|
+
for await (const item of iter) {
|
|
631
650
|
if (performance.now() - startTime >= LSN_TIMEOUT_SECONDS * 1000) {
|
|
632
651
|
break;
|
|
633
652
|
}
|
|
@@ -635,26 +654,26 @@ export class MongoSnapshotter {
|
|
|
635
654
|
await this.checkpointImplementation.createBatchCheckpoint();
|
|
636
655
|
lastCheckpointCreated = performance.now();
|
|
637
656
|
}
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
// Observing our own barrier has set the coordinate (the barrier event
|
|
652
|
-
// carries the sentinel counter directly), so the LSN is ready.
|
|
653
|
-
return this.checkpointImplementation.event.lsn(changeDocument);
|
|
657
|
+
if (item.type == 'progress') {
|
|
658
|
+
batchesSeen += 1;
|
|
659
|
+
continue;
|
|
660
|
+
}
|
|
661
|
+
const changeDocument = item.event;
|
|
662
|
+
const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
|
|
663
|
+
|
|
664
|
+
if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
|
|
665
|
+
const kind = this.checkpointImplementation.event.observe(changeDocument);
|
|
666
|
+
if (kind != 'own-barrier') {
|
|
667
|
+
// Standalone events still feed the implementation's coordinate via
|
|
668
|
+
// event.observe above; we only resolve on our own barrier.
|
|
669
|
+
continue;
|
|
654
670
|
}
|
|
655
|
-
|
|
656
|
-
|
|
671
|
+
// Observing our own barrier has set the coordinate (the barrier event
|
|
672
|
+
// carries the sentinel counter directly), so the LSN is ready.
|
|
673
|
+
return this.checkpointImplementation.event.lsn(changeDocument);
|
|
657
674
|
}
|
|
675
|
+
|
|
676
|
+
eventsSeen += 1;
|
|
658
677
|
}
|
|
659
678
|
|
|
660
679
|
// Could happen if there is a very large replication lag?
|
|
@@ -668,12 +687,13 @@ export class MongoSnapshotter {
|
|
|
668
687
|
* Given a snapshot LSN, validate that we can read from it, by opening a change stream.
|
|
669
688
|
*/
|
|
670
689
|
private async validateSnapshotLsn(lsn: string) {
|
|
671
|
-
const stream = this.
|
|
690
|
+
const stream = this.openChangeStream({
|
|
672
691
|
lsn,
|
|
673
692
|
maxAwaitTimeMS: 0,
|
|
674
|
-
filters: this.getSourceNamespaceFilters()
|
|
693
|
+
filters: this.getSourceNamespaceFilters(),
|
|
694
|
+
signal: this.abortSignal
|
|
675
695
|
});
|
|
676
|
-
for await (const
|
|
696
|
+
for await (const _item of stream) {
|
|
677
697
|
break;
|
|
678
698
|
}
|
|
679
699
|
}
|
|
@@ -722,73 +742,31 @@ export class MongoSnapshotter {
|
|
|
722
742
|
return { $match: nsFilter, multipleDatabases };
|
|
723
743
|
}
|
|
724
744
|
|
|
725
|
-
private
|
|
745
|
+
private openChangeStream(options: {
|
|
726
746
|
lsn: string | null;
|
|
727
747
|
maxAwaitTimeMS?: number;
|
|
728
748
|
batchSize?: number;
|
|
729
749
|
filters: { $match: any; multipleDatabases: boolean };
|
|
730
750
|
signal?: AbortSignal;
|
|
731
751
|
tracer?: PerformanceTracer<'changestream'>;
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
fullDocument
|
|
751
|
-
};
|
|
752
|
-
if (!this.isDocumentDb) {
|
|
753
|
-
// DocumentDB does not support showExpandedEvents.
|
|
754
|
-
streamOptions.showExpandedEvents = true;
|
|
755
|
-
}
|
|
756
|
-
const pipeline: mongo.Document[] = [{ $changeStream: streamOptions }, { $match: options.filters.$match }];
|
|
757
|
-
if (!this.isDocumentDb) {
|
|
758
|
-
// DocumentDB does not support $changeStreamSplitLargeEvent.
|
|
759
|
-
pipeline.push({ $changeStreamSplitLargeEvent: {} });
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
// Only one of these options can be supplied at a time.
|
|
763
|
-
if (resumeAfter) {
|
|
764
|
-
streamOptions.resumeAfter = resumeAfter;
|
|
765
|
-
} else if (startAfter != null) {
|
|
766
|
-
// Legacy: We don't persist lsns without resumeTokens anymore, but we do still handle the
|
|
767
|
-
// case if we have an old one. The sentinel implementation never produces a startAfter,
|
|
768
|
-
// and a fresh DocumentDB stream opens from "now" with neither option set.
|
|
769
|
-
streamOptions.startAtOperationTime = startAfter;
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
let watchDb: mongo.Db;
|
|
773
|
-
if (this.isDocumentDb || options.filters.multipleDatabases) {
|
|
774
|
-
// DocumentDB only supports cluster-level change streams.
|
|
775
|
-
// Requires readAnyDatabase@admin on Atlas.
|
|
776
|
-
watchDb = this.client.db('admin');
|
|
777
|
-
streamOptions.allChangesForCluster = true;
|
|
778
|
-
} else {
|
|
779
|
-
// Same general result, but requires less permissions than the above
|
|
780
|
-
watchDb = this.defaultDb;
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
const maxAwaitTimeMS = options.maxAwaitTimeMS ?? this.maxAwaitTimeMS;
|
|
784
|
-
|
|
785
|
-
return rawChangeStream(watchDb, pipeline, {
|
|
786
|
-
batchSize: options.batchSize ?? this.snapshotChunkLength,
|
|
787
|
-
maxAwaitTimeMS,
|
|
788
|
-
maxTimeMS: this.changeStreamTimeout,
|
|
789
|
-
signal: options.signal,
|
|
790
|
-
logger: this.logger,
|
|
791
|
-
tracer: options.tracer
|
|
752
|
+
onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
|
|
753
|
+
}): AsyncIterableIterator<MongoReplicationStreamItem> {
|
|
754
|
+
return openMongoReplicationStream({
|
|
755
|
+
db: this.defaultDb,
|
|
756
|
+
queryProvider: this.queryProvider,
|
|
757
|
+
namespaceFilter: options.filters,
|
|
758
|
+
isDocumentDb: this.isDocumentDb,
|
|
759
|
+
usePostImages: this.usePostImages,
|
|
760
|
+
position: options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null,
|
|
761
|
+
onBatch: options.onBatch,
|
|
762
|
+
options: {
|
|
763
|
+
batchSize: options.batchSize ?? this.snapshotChunkLength,
|
|
764
|
+
maxAwaitTimeMS: options.maxAwaitTimeMS ?? this.maxAwaitTimeMS,
|
|
765
|
+
maxTimeMS: this.changeStreamTimeout,
|
|
766
|
+
signal: options.signal,
|
|
767
|
+
logger: this.logger,
|
|
768
|
+
tracer: options.tracer
|
|
769
|
+
}
|
|
792
770
|
});
|
|
793
771
|
}
|
|
794
772
|
|
|
@@ -364,7 +364,7 @@ type MapRawDocument<T> = T extends unknown ? { [K in keyof T]: Rawify<T[K]> } :
|
|
|
364
364
|
export type ProjectedChangeStreamDocument =
|
|
365
365
|
| Omit<mongo.ChangeStreamDropDocument, 'wallTime' | 'collectionUUID'>
|
|
366
366
|
| Omit<mongo.ChangeStreamRenameDocument, 'wallTime'>
|
|
367
|
-
| Omit<mongo.ChangeStreamDeleteDocument<Buffer>, 'wallTime'>
|
|
367
|
+
| Omit<mongo.ChangeStreamDeleteDocument<Buffer>, 'wallTime' | 'collectionUUID'>
|
|
368
368
|
| Omit<mongo.ChangeStreamInsertDocument<Buffer>, 'wallTime'>
|
|
369
369
|
| Omit<mongo.ChangeStreamUpdateDocument<Buffer>, 'wallTime' | 'updateDescription'>
|
|
370
370
|
| Omit<mongo.ChangeStreamReplaceDocument<Buffer>, 'wallTime'>;
|
|
@@ -432,7 +432,7 @@ export function mapChangeStreamError(e: unknown) {
|
|
|
432
432
|
} else if (
|
|
433
433
|
isMongoServerError(e) &&
|
|
434
434
|
e.codeName == 'NoMatchingDocument' &&
|
|
435
|
-
e.errmsg?.includes('post-image was not found')
|
|
435
|
+
(e.errmsg?.includes('post-image was not found') || e.errmsg?.includes('pre-image was not found'))
|
|
436
436
|
) {
|
|
437
437
|
throw new ChangeStreamInvalidatedError(e.errmsg, e);
|
|
438
438
|
} else if (isMongoServerError(e) && e.hasErrorLabel('NonResumableChangeStreamError')) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { applyRowContext, CompatibilityContext, SqliteRow } from '@powersync/service-sync-rules';
|
|
2
|
+
import { applyRowContext, CompatibilityContext, CompatibilityOption, SqliteRow } from '@powersync/service-sync-rules';
|
|
3
3
|
import { bufferToSqlite, DateRenderMode, getDateRenderMode, parseDocumentId } from './bufferToSqlite.js';
|
|
4
4
|
import { constructAfterRecord } from './MongoRelation.js';
|
|
5
5
|
|
|
@@ -46,13 +46,15 @@ export class LegacySourceRowConverter implements SourceRowConverter {
|
|
|
46
46
|
|
|
47
47
|
export class DirectSourceRowConverter implements SourceRowConverter {
|
|
48
48
|
private readonly dateRenderMode: DateRenderMode;
|
|
49
|
+
private readonly fixedBooleansInJson: boolean;
|
|
49
50
|
|
|
50
51
|
constructor(compatibilityContext: CompatibilityContext) {
|
|
51
52
|
this.dateRenderMode = getDateRenderMode(compatibilityContext);
|
|
53
|
+
this.fixedBooleansInJson = compatibilityContext.isEnabled(CompatibilityOption.fixedBooleanInJson);
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
rawToSqliteRow(source: Buffer): { row: SqliteRow; replicaId: any } {
|
|
55
|
-
const row = bufferToSqlite(source, this.dateRenderMode);
|
|
57
|
+
const row = bufferToSqlite(source, this.dateRenderMode, this.fixedBooleansInJson);
|
|
56
58
|
const replicaId = parseDocumentId(source).id;
|
|
57
59
|
return { row, replicaId };
|
|
58
60
|
}
|
|
@@ -80,12 +80,16 @@ const SHARED_WRITER = new JsonBufferWriter();
|
|
|
80
80
|
*
|
|
81
81
|
* @param bytes the source BSON bytes
|
|
82
82
|
* @param dateRenderMode derive using getDateRenderMode(compatibilityContext)
|
|
83
|
+
* @param fixedBooleansInJson whether to serialize booleans in nested JSON as booleans, derived from compatibility
|
|
84
|
+
* context.
|
|
83
85
|
*
|
|
84
86
|
* @returns a SqliteRow
|
|
85
87
|
*/
|
|
86
|
-
export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode): SqliteRow {
|
|
88
|
+
export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode, fixedBooleansInJson: boolean): SqliteRow {
|
|
87
89
|
const row: SqliteRow = {};
|
|
88
90
|
const jsonWriter = SHARED_WRITER;
|
|
91
|
+
const options: SerializeToJsonOptions = { dateRenderMode, fixedBooleans: fixedBooleansInJson };
|
|
92
|
+
|
|
89
93
|
// BSON documents are length-prefixed and null-terminated. We parse directly
|
|
90
94
|
// from raw bytes, so structural validation happens here rather than in the
|
|
91
95
|
// upstream BSON decoder.
|
|
@@ -112,14 +116,14 @@ export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode): S
|
|
|
112
116
|
}
|
|
113
117
|
case BSON_TYPE_ARRAY: {
|
|
114
118
|
jsonWriter.reset();
|
|
115
|
-
const result = serializeNestedArrayToJson(bytes, offset, 0, jsonWriter,
|
|
119
|
+
const result = serializeNestedArrayToJson(bytes, offset, 0, jsonWriter, options);
|
|
116
120
|
row[key] = jsonWriter.toString();
|
|
117
121
|
offset = result.nextOffset;
|
|
118
122
|
break;
|
|
119
123
|
}
|
|
120
124
|
case BSON_TYPE_DOCUMENT: {
|
|
121
125
|
jsonWriter.reset();
|
|
122
|
-
const result = serializeNestedObjectToJson(bytes, offset, 0, jsonWriter,
|
|
126
|
+
const result = serializeNestedObjectToJson(bytes, offset, 0, jsonWriter, options);
|
|
123
127
|
row[key] = jsonWriter.toString();
|
|
124
128
|
offset = result.nextOffset;
|
|
125
129
|
break;
|
|
@@ -195,7 +199,7 @@ export function bufferToSqlite(bytes: Buffer, dateRenderMode: DateRenderMode): S
|
|
|
195
199
|
}
|
|
196
200
|
case BSON_TYPE_CODE_WITH_SCOPE: {
|
|
197
201
|
jsonWriter.reset();
|
|
198
|
-
const nextOffset = writeCodeWithScopeJson(bytes, offset, 0, jsonWriter,
|
|
202
|
+
const nextOffset = writeCodeWithScopeJson(bytes, offset, 0, jsonWriter, options);
|
|
199
203
|
row[key] = jsonWriter.toString();
|
|
200
204
|
offset = nextOffset;
|
|
201
205
|
break;
|
|
@@ -437,12 +441,17 @@ function skipBsonValue(bytes: Buffer, offset: number, type: number) {
|
|
|
437
441
|
}
|
|
438
442
|
}
|
|
439
443
|
|
|
444
|
+
interface SerializeToJsonOptions {
|
|
445
|
+
dateRenderMode: DateRenderMode;
|
|
446
|
+
fixedBooleans: boolean;
|
|
447
|
+
}
|
|
448
|
+
|
|
440
449
|
function serializeNestedObjectToJson(
|
|
441
450
|
bytes: Buffer,
|
|
442
451
|
offset: number,
|
|
443
452
|
depth: number,
|
|
444
453
|
writer: JsonBufferWriter,
|
|
445
|
-
|
|
454
|
+
options: SerializeToJsonOptions
|
|
446
455
|
): { nextOffset: number } {
|
|
447
456
|
if (depth > NESTED_DEPTH_LIMIT) {
|
|
448
457
|
throw new Error(`json nested object depth exceeds the limit of ${NESTED_DEPTH_LIMIT}`);
|
|
@@ -475,7 +484,7 @@ function serializeNestedObjectToJson(
|
|
|
475
484
|
type,
|
|
476
485
|
depth,
|
|
477
486
|
writer,
|
|
478
|
-
|
|
487
|
+
options
|
|
479
488
|
);
|
|
480
489
|
cursor = afterValue;
|
|
481
490
|
// Malformed BSON must fail fast instead of getting the parser stuck on the
|
|
@@ -499,7 +508,7 @@ function serializeNestedArrayToJson(
|
|
|
499
508
|
offset: number,
|
|
500
509
|
depth: number,
|
|
501
510
|
writer: JsonBufferWriter,
|
|
502
|
-
|
|
511
|
+
options: SerializeToJsonOptions
|
|
503
512
|
): { nextOffset: number } {
|
|
504
513
|
if (depth > NESTED_DEPTH_LIMIT) {
|
|
505
514
|
throw new Error(`json nested object depth exceeds the limit of ${NESTED_DEPTH_LIMIT}`);
|
|
@@ -527,7 +536,7 @@ function serializeNestedArrayToJson(
|
|
|
527
536
|
type,
|
|
528
537
|
depth,
|
|
529
538
|
writer,
|
|
530
|
-
|
|
539
|
+
options
|
|
531
540
|
);
|
|
532
541
|
cursor = afterValue;
|
|
533
542
|
assertAdvanced(previousCursor, cursor);
|
|
@@ -547,7 +556,7 @@ function serializeNestedElementValue(
|
|
|
547
556
|
type: number,
|
|
548
557
|
depth: number,
|
|
549
558
|
writer: JsonBufferWriter,
|
|
550
|
-
|
|
559
|
+
options: SerializeToJsonOptions
|
|
551
560
|
): { nextOffset: number; defined: boolean } {
|
|
552
561
|
switch (type) {
|
|
553
562
|
case BSON_TYPE_DOUBLE: // Double
|
|
@@ -555,9 +564,9 @@ function serializeNestedElementValue(
|
|
|
555
564
|
case BSON_TYPE_STRING: // String
|
|
556
565
|
return serializeNestedStringElement(bytes, offset, writer);
|
|
557
566
|
case BSON_TYPE_DOCUMENT: // Embedded document
|
|
558
|
-
return serializeNestedObjectElement(bytes, offset, depth, writer,
|
|
567
|
+
return serializeNestedObjectElement(bytes, offset, depth, writer, options);
|
|
559
568
|
case BSON_TYPE_ARRAY: // Array
|
|
560
|
-
return serializeNestedArrayElement(bytes, offset, depth, writer,
|
|
569
|
+
return serializeNestedArrayElement(bytes, offset, depth, writer, options);
|
|
561
570
|
case BSON_TYPE_BINARY: // Binary
|
|
562
571
|
return serializeNestedBinaryElement(bytes, offset, writer);
|
|
563
572
|
case BSON_TYPE_UNDEFINED: // Undefined
|
|
@@ -567,11 +576,22 @@ function serializeNestedElementValue(
|
|
|
567
576
|
writer.writeQuotedHexLower(bytes, offset, 12);
|
|
568
577
|
return { nextOffset: offset + 12, defined: true };
|
|
569
578
|
}
|
|
570
|
-
case BSON_TYPE_BOOLEAN:
|
|
571
|
-
|
|
579
|
+
case BSON_TYPE_BOOLEAN: {
|
|
580
|
+
// Boolean
|
|
581
|
+
const value = !!bytes[offset];
|
|
582
|
+
|
|
583
|
+
if (options.fixedBooleans) {
|
|
584
|
+
const str = value ? 'true' : 'false';
|
|
585
|
+
|
|
586
|
+
writer.writeAscii(str);
|
|
587
|
+
} else {
|
|
588
|
+
writer.writeByte(value ? BYTE_ONE : BYTE_ZERO);
|
|
589
|
+
}
|
|
590
|
+
|
|
572
591
|
return { nextOffset: offset + 1, defined: true };
|
|
592
|
+
}
|
|
573
593
|
case BSON_TYPE_UTC_DATETIME: // UTC datetime
|
|
574
|
-
return serializeNestedDateTimeElement(bytes, offset, writer, dateRenderMode);
|
|
594
|
+
return serializeNestedDateTimeElement(bytes, offset, writer, options.dateRenderMode);
|
|
575
595
|
case BSON_TYPE_NULL: // Null
|
|
576
596
|
case BSON_TYPE_MIN_KEY: // MinKey
|
|
577
597
|
case BSON_TYPE_MAX_KEY: // MaxKey
|
|
@@ -586,7 +606,7 @@ function serializeNestedElementValue(
|
|
|
586
606
|
case BSON_TYPE_SYMBOL: // Symbol
|
|
587
607
|
return serializeNestedSymbolElement(bytes, offset, writer);
|
|
588
608
|
case BSON_TYPE_CODE_WITH_SCOPE: // JavaScript code with scope
|
|
589
|
-
return serializeNestedCodeWithScopeElement(bytes, offset, depth, writer,
|
|
609
|
+
return serializeNestedCodeWithScopeElement(bytes, offset, depth, writer, options);
|
|
590
610
|
case BSON_TYPE_INT32: {
|
|
591
611
|
// Int32
|
|
592
612
|
writer.writeAscii(String(readInt32LE(bytes, offset)));
|
|
@@ -641,9 +661,9 @@ function serializeNestedObjectElement(
|
|
|
641
661
|
offset: number,
|
|
642
662
|
depth: number,
|
|
643
663
|
writer: JsonBufferWriter,
|
|
644
|
-
|
|
664
|
+
options: SerializeToJsonOptions
|
|
645
665
|
): { nextOffset: number; defined: boolean } {
|
|
646
|
-
const result = serializeNestedObjectToJson(bytes, offset, depth + 1, writer,
|
|
666
|
+
const result = serializeNestedObjectToJson(bytes, offset, depth + 1, writer, options);
|
|
647
667
|
return { nextOffset: result.nextOffset, defined: true };
|
|
648
668
|
}
|
|
649
669
|
|
|
@@ -652,9 +672,9 @@ function serializeNestedArrayElement(
|
|
|
652
672
|
offset: number,
|
|
653
673
|
depth: number,
|
|
654
674
|
writer: JsonBufferWriter,
|
|
655
|
-
|
|
675
|
+
options: SerializeToJsonOptions
|
|
656
676
|
): { nextOffset: number; defined: boolean } {
|
|
657
|
-
const result = serializeNestedArrayToJson(bytes, offset, depth + 1, writer,
|
|
677
|
+
const result = serializeNestedArrayToJson(bytes, offset, depth + 1, writer, options);
|
|
658
678
|
return { nextOffset: result.nextOffset, defined: true };
|
|
659
679
|
}
|
|
660
680
|
|
|
@@ -791,14 +811,14 @@ function writeCodeWithScopeJson(
|
|
|
791
811
|
offset: number,
|
|
792
812
|
depth: number,
|
|
793
813
|
writer: JsonBufferWriter,
|
|
794
|
-
|
|
814
|
+
options: SerializeToJsonOptions
|
|
795
815
|
) {
|
|
796
816
|
const totalLength = readInt32LE(bytes, offset);
|
|
797
817
|
const { value: code, nextOffset: afterCode } = readBsonString(bytes, offset + 4);
|
|
798
818
|
writer.writeAscii('{"code":');
|
|
799
819
|
writer.writeQuotedJsonString(code);
|
|
800
820
|
writer.writeAscii(',"scope":');
|
|
801
|
-
serializeNestedObjectToJson(bytes, afterCode, depth + 1, writer,
|
|
821
|
+
serializeNestedObjectToJson(bytes, afterCode, depth + 1, writer, options);
|
|
802
822
|
writer.writeByte(BYTE_RBRACE);
|
|
803
823
|
// code_w_scope carries its own total byte length, so we trust that wrapper
|
|
804
824
|
// rather than reconstructing the end position from the nested scope.
|
|
@@ -885,10 +905,10 @@ function serializeNestedCodeWithScopeElement(
|
|
|
885
905
|
offset: number,
|
|
886
906
|
depth: number,
|
|
887
907
|
writer: JsonBufferWriter,
|
|
888
|
-
|
|
908
|
+
options: SerializeToJsonOptions
|
|
889
909
|
): { nextOffset: number; defined: boolean } {
|
|
890
910
|
return {
|
|
891
|
-
nextOffset: writeCodeWithScopeJson(bytes, offset, depth, writer,
|
|
911
|
+
nextOffset: writeCodeWithScopeJson(bytes, offset, depth, writer, options),
|
|
892
912
|
defined: true
|
|
893
913
|
};
|
|
894
914
|
}
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
export * from './ChangeStream.js';
|
|
2
2
|
export * from './ChangeStreamReplicationJob.js';
|
|
3
3
|
export * from './ChangeStreamReplicator.js';
|
|
4
|
+
export * from './ConnectionManagerFactory.js';
|
|
4
5
|
export * from './internal-mongodb-utils.js';
|
|
6
|
+
export * from './MongoErrorRateLimiter.js';
|
|
5
7
|
export * from './MongoRelation.js';
|
|
8
|
+
export * from './MongoReplicationQueryProvider.js';
|
|
9
|
+
export * from './MongoReplicationStream.js';
|
|
10
|
+
export * from './MongoSnapshotter.js';
|
|
6
11
|
export * from './RawChangeStream.js';
|
|
12
|
+
export { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
7
13
|
export * from './SourceRowConverter.js';
|