@powersync/service-module-mongodb 0.0.0-dev-20261005120023 → 0.0.0-dev-20261015071128
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -7
- package/benchmarks/env.ts +11 -0
- package/benchmarks/replication_branch_throughput.benchmark.ts +240 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +6 -0
- package/dist/api/MongoRouteAPIAdapter.js +21 -2
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/module/MongoModule.js +1 -0
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +3 -1
- package/dist/replication/ChangeStream.js +173 -150
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +17 -9
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/MongoReplicationQueryProvider.d.ts +9 -11
- package/dist/replication/MongoReplicationQueryProvider.js.map +1 -1
- package/dist/replication/MongoReplicationStream.d.ts +37 -48
- package/dist/replication/MongoReplicationStream.js +157 -30
- package/dist/replication/MongoReplicationStream.js.map +1 -1
- package/dist/replication/MongoReplicationStreamTypes.d.ts +157 -0
- package/dist/replication/MongoReplicationStreamTypes.js +19 -0
- package/dist/replication/MongoReplicationStreamTypes.js.map +1 -0
- package/dist/replication/MongoSnapshotter.js +45 -25
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/replication-index.d.ts +1 -0
- package/dist/replication/replication-index.js +1 -0
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/test-utils/ChangeStreamTestContext.js +2 -3
- package/dist/test-utils/ChangeStreamTestContext.js.map +1 -1
- package/package.json +11 -10
- package/src/api/MongoRouteAPIAdapter.ts +26 -2
- package/src/module/MongoModule.ts +1 -0
- package/src/replication/ChangeStream.ts +197 -174
- package/src/replication/ChangeStreamReplicationJob.ts +16 -9
- package/src/replication/MongoReplicationQueryProvider.ts +9 -11
- package/src/replication/MongoReplicationStream.ts +196 -83
- package/src/replication/MongoReplicationStreamTypes.ts +169 -0
- package/src/replication/MongoSnapshotter.ts +49 -28
- package/src/replication/replication-index.ts +1 -0
- package/src/test-utils/ChangeStreamTestContext.ts +2 -2
- package/test/src/change_event_memory.test.ts +33 -0
- package/test/src/change_stream_abort.test.ts +110 -0
- package/test/src/filtering_example.test.ts +65 -0
- package/test/src/filtering_lifecycle.test.ts +101 -0
- package/test/src/query_provider_availability.test.ts +77 -0
- package/test/src/query_provider_diagnostics.test.ts +55 -0
- package/test/src/replication_stream.test.ts +411 -34
- package/test/src/stream_progress.test.ts +62 -44
- package/test/tsconfig.json +2 -2
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.benchmark.config.ts +15 -0
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
SourceTable,
|
|
19
19
|
storage
|
|
20
20
|
} from '@powersync/service-core';
|
|
21
|
-
import { HydratedSyncConfig } from '@powersync/service-sync-rules';
|
|
21
|
+
import { hasMongoFilterExpressions, HydratedSyncConfig } from '@powersync/service-sync-rules';
|
|
22
22
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
23
23
|
import { performance } from 'node:perf_hooks';
|
|
24
24
|
import { PostImagesOption } from '../types/types.js';
|
|
@@ -32,14 +32,15 @@ import {
|
|
|
32
32
|
MongoReplicationQueryProvider,
|
|
33
33
|
MongoReplicationQueryProviderFactory
|
|
34
34
|
} from './MongoReplicationQueryProvider.js';
|
|
35
|
-
import {
|
|
35
|
+
import { openMongoReplicationStream } from './MongoReplicationStream.js';
|
|
36
|
+
import type { MongoReplicationStreamBatch } from './MongoReplicationStreamTypes.js';
|
|
36
37
|
import { MongoSnapshotter, MongoSnapshotterHooks } from './MongoSnapshotter.js';
|
|
37
38
|
import { ChangeStreamBatch, ProjectedChangeStreamDocument } from './RawChangeStream.js';
|
|
38
39
|
import { CHECKPOINTS_COLLECTION, detectDocumentDb, timestampToDate } from './replication-utils.js';
|
|
39
40
|
import { DirectSourceRowConverter, SourceRowConverter } from './SourceRowConverter.js';
|
|
40
41
|
export interface ChangeStreamOptions {
|
|
41
|
-
connections: MongoManager;
|
|
42
42
|
storage: storage.SyncRulesBucketStorage;
|
|
43
|
+
connections: MongoManager;
|
|
43
44
|
metrics: MetricsEngine;
|
|
44
45
|
abort_signal: AbortSignal;
|
|
45
46
|
/**
|
|
@@ -124,6 +125,7 @@ export class ChangeStream {
|
|
|
124
125
|
|
|
125
126
|
private readonly sourceRowConverter: SourceRowConverter;
|
|
126
127
|
private readonly queryProvider: MongoReplicationQueryProvider;
|
|
128
|
+
private readonly hasQueryProviderFactory: boolean;
|
|
127
129
|
|
|
128
130
|
private keepaliveIntervalMs: number;
|
|
129
131
|
|
|
@@ -136,7 +138,7 @@ export class ChangeStream {
|
|
|
136
138
|
private lastPersistedResumeTimestamp = 0;
|
|
137
139
|
private lastBatchCheckpoint = 0;
|
|
138
140
|
|
|
139
|
-
constructor(options: ChangeStreamOptions) {
|
|
141
|
+
constructor(private readonly options: ChangeStreamOptions) {
|
|
140
142
|
this.storage = options.storage;
|
|
141
143
|
this.metrics = options.metrics;
|
|
142
144
|
this.group_id = options.storage.replicationStreamId;
|
|
@@ -156,6 +158,7 @@ export class ChangeStream {
|
|
|
156
158
|
this.sync_rules = options.storage.getParsedSyncRules({
|
|
157
159
|
defaultSchema: this.defaultDb.databaseName
|
|
158
160
|
});
|
|
161
|
+
this.hasQueryProviderFactory = options.createReplicationQueryProvider != null;
|
|
159
162
|
this.queryProvider =
|
|
160
163
|
options.createReplicationQueryProvider?.({
|
|
161
164
|
syncConfig: this.sync_rules,
|
|
@@ -501,6 +504,17 @@ export class ChangeStream {
|
|
|
501
504
|
}
|
|
502
505
|
|
|
503
506
|
private async initReplication() {
|
|
507
|
+
// A previously deployed config may outlive the module that provided pre-filtering.
|
|
508
|
+
// Throw inside replicate() so the failure is persisted in replication diagnostics.
|
|
509
|
+
if (
|
|
510
|
+
!this.hasQueryProviderFactory &&
|
|
511
|
+
hasMongoFilterExpressions(this.sync_rules.sourceTableConfig, this.connections.connectionTag)
|
|
512
|
+
) {
|
|
513
|
+
throw new ServiceError(
|
|
514
|
+
ErrorCode.PSYNC_S1348,
|
|
515
|
+
'This sync config requires MongoDB pre-filtering, but no query provider factory is registered. Restore the required module, or remove the filters and deploy a new sync config to restart replication.'
|
|
516
|
+
);
|
|
517
|
+
}
|
|
504
518
|
const result = await this.snapshotter.checkSlot();
|
|
505
519
|
await this.snapshotter.setupCheckpointsCollection();
|
|
506
520
|
if (result.needsInitialSync) {
|
|
@@ -544,7 +558,7 @@ export class ChangeStream {
|
|
|
544
558
|
signal?: AbortSignal;
|
|
545
559
|
tracer?: PerformanceTracer<'changestream'>;
|
|
546
560
|
onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
|
|
547
|
-
}): AsyncIterableIterator<
|
|
561
|
+
}): AsyncIterableIterator<MongoReplicationStreamBatch> {
|
|
548
562
|
return openMongoReplicationStream({
|
|
549
563
|
db: this.defaultDb,
|
|
550
564
|
queryProvider: this.queryProvider,
|
|
@@ -611,6 +625,7 @@ export class ChangeStream {
|
|
|
611
625
|
let processingSpan: ReturnType<typeof tracer.span> | undefined;
|
|
612
626
|
let receivedBytes = 0;
|
|
613
627
|
let changesSinceProgress = 0;
|
|
628
|
+
let filteredSinceProgress: number | null = null;
|
|
614
629
|
const stream = this.openChangeStream({
|
|
615
630
|
lsn: resumeFromLsn,
|
|
616
631
|
filters,
|
|
@@ -634,13 +649,180 @@ export class ChangeStream {
|
|
|
634
649
|
let lastKeepalive = performance.now();
|
|
635
650
|
let lastTxnKey: string | null = null;
|
|
636
651
|
|
|
637
|
-
for await (const
|
|
652
|
+
streamLoop: for await (const sourceBatch of stream) {
|
|
653
|
+
if (this.abortSignal.aborted) break;
|
|
654
|
+
for (const { event: changeDocument, hasBufferedChanges } of sourceBatch.events) {
|
|
655
|
+
if (this.abortSignal.aborted) break streamLoop;
|
|
656
|
+
this.touch();
|
|
657
|
+
changesSinceProgress++;
|
|
658
|
+
const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
|
|
659
|
+
|
|
660
|
+
if (ns?.coll == CHECKPOINTS_COLLECTION) {
|
|
661
|
+
/**
|
|
662
|
+
* Dropping the database does not provide an `invalidate` event.
|
|
663
|
+
* We typically would receive `drop` events for the collection which we
|
|
664
|
+
* would process below.
|
|
665
|
+
*
|
|
666
|
+
* However we don't commit the LSN after collections are dropped.
|
|
667
|
+
* This prevents the `startAfter` or `resumeToken` from advancing past the drop events.
|
|
668
|
+
* The stream also closes after the drop events.
|
|
669
|
+
* This causes an infinite loop of processing the collection drop events.
|
|
670
|
+
*
|
|
671
|
+
* This check here invalidates the change stream if our `_powersync_checkpoints` collection
|
|
672
|
+
* is dropped. This allows for detecting when the DB is dropped.
|
|
673
|
+
*/
|
|
674
|
+
if (changeDocument.operationType == 'drop') {
|
|
675
|
+
throw new ChangeStreamInvalidatedError(
|
|
676
|
+
'Internal collections have been dropped',
|
|
677
|
+
new Error('_powersync_checkpoints collection was dropped')
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
if (
|
|
682
|
+
!(
|
|
683
|
+
changeDocument.operationType == 'insert' ||
|
|
684
|
+
changeDocument.operationType == 'update' ||
|
|
685
|
+
changeDocument.operationType == 'replace'
|
|
686
|
+
)
|
|
687
|
+
) {
|
|
688
|
+
continue;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
// We handle two types of checkpoint events:
|
|
692
|
+
// 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
|
|
693
|
+
// immediately, regardless of where they were created.
|
|
694
|
+
// 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
|
|
695
|
+
// limiting of commits, so we specifically want to exclude checkpoints from other streams.
|
|
696
|
+
//
|
|
697
|
+
// It may be useful to also throttle commits due to standalone checkpoints in the future.
|
|
698
|
+
// However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
|
|
699
|
+
|
|
700
|
+
const kind = this.checkpointImplementation.event.observe(changeDocument);
|
|
701
|
+
|
|
702
|
+
if (kind == 'foreign') {
|
|
703
|
+
// Another stream's barrier - ignore.
|
|
704
|
+
continue;
|
|
705
|
+
} else if (kind == 'standalone') {
|
|
706
|
+
// Standalone / write checkpoint received.
|
|
707
|
+
// When we are caught up, commit immediately to keep write checkpoint latency low.
|
|
708
|
+
// Once there is already a batch checkpoint pending, or the driver has buffered more
|
|
709
|
+
// change stream events, collapse standalone checkpoints into the normal batch
|
|
710
|
+
// checkpoint flow to avoid commit churn under sustained load.
|
|
711
|
+
if (hasBufferedChanges && waitForCheckpointLsn == null) {
|
|
712
|
+
// Buffered changes - create a new batch checkpoint to rate limit commits
|
|
713
|
+
using _ = tracer.span('source_checkpoint');
|
|
714
|
+
waitForCheckpointLsn = await this.createBatchCheckpoint();
|
|
715
|
+
continue;
|
|
716
|
+
} else if (waitForCheckpointLsn != null) {
|
|
717
|
+
// Skip this checkpoint - wait for the batch checkpoint.
|
|
718
|
+
continue;
|
|
719
|
+
} else {
|
|
720
|
+
// No buffered changes, and no batch checkpoint pending - commit immediately.
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
// kind == 'own-barrier' falls through to commit.
|
|
724
|
+
|
|
725
|
+
const lsn = this.checkpointImplementation.event.lsn(changeDocument);
|
|
726
|
+
|
|
727
|
+
if (
|
|
728
|
+
waitForCheckpointLsn != null &&
|
|
729
|
+
this.checkpointImplementation.event.resolvesBarrier(waitForCheckpointLsn, changeDocument)
|
|
730
|
+
) {
|
|
731
|
+
waitForCheckpointLsn = null;
|
|
732
|
+
}
|
|
733
|
+
const { checkpointBlocked, checkpointCreated } = await batch.commit(lsn, {
|
|
734
|
+
oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
if (!checkpointBlocked || checkpointCreated) {
|
|
738
|
+
this.replicationLag.markCommitted();
|
|
739
|
+
}
|
|
740
|
+
} else if (
|
|
741
|
+
changeDocument.operationType == 'insert' ||
|
|
742
|
+
changeDocument.operationType == 'update' ||
|
|
743
|
+
changeDocument.operationType == 'replace' ||
|
|
744
|
+
changeDocument.operationType == 'delete'
|
|
745
|
+
) {
|
|
746
|
+
if (waitForCheckpointLsn == null) {
|
|
747
|
+
using _ = tracer.span('source_checkpoint');
|
|
748
|
+
waitForCheckpointLsn = await this.createBatchCheckpoint();
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
|
|
752
|
+
const tables = await this.getRelations(batch, rel, {
|
|
753
|
+
// In most cases, we should not need to snapshot this. But if this is the first time we see the collection
|
|
754
|
+
// for whatever reason, then we do need to snapshot it.
|
|
755
|
+
// This may result in some duplicate operations when a collection is created for the first time after
|
|
756
|
+
// sync config was deployed.
|
|
757
|
+
snapshot: true
|
|
758
|
+
});
|
|
759
|
+
const tablesToReplicate = tables.filter((table) => table.syncAny);
|
|
760
|
+
if (tablesToReplicate.length > 0) {
|
|
761
|
+
this.replicationLag.trackUncommittedChange(
|
|
762
|
+
// Standard MongoDB uses clusterTime, unchanged. DocumentDB has no
|
|
763
|
+
// clusterTime, so fall back to wallTime there for the lag metric.
|
|
764
|
+
changeDocument.clusterTime != null
|
|
765
|
+
? timestampToDate(changeDocument.clusterTime)
|
|
766
|
+
: ((changeDocument as any).wallTime ?? null)
|
|
767
|
+
);
|
|
768
|
+
|
|
769
|
+
const transactionKeyValue = transactionKey(changeDocument);
|
|
770
|
+
|
|
771
|
+
if (transactionKeyValue == null || lastTxnKey != transactionKeyValue) {
|
|
772
|
+
// Very crude metric for counting transactions replicated.
|
|
773
|
+
// We ignore operations other than basic CRUD, and ignore changes to _powersync_checkpoints.
|
|
774
|
+
// Individual writes may not have a txnNumber, in which case we count them as separate transactions.
|
|
775
|
+
lastTxnKey = transactionKeyValue;
|
|
776
|
+
transactionsReplicatedMetric.add(1);
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
for (const table of tablesToReplicate) {
|
|
780
|
+
await this.writeChange(batch, table, changeDocument);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
} else if (changeDocument.operationType == 'drop') {
|
|
784
|
+
const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
|
|
785
|
+
const tables = await this.getRelations(batch, rel, {
|
|
786
|
+
// We're "dropping" this collection, so never snapshot it.
|
|
787
|
+
snapshot: false
|
|
788
|
+
});
|
|
789
|
+
const tablesToDrop = tables.filter((table) => table.syncAny);
|
|
790
|
+
if (tablesToDrop.length > 0) {
|
|
791
|
+
await batch.drop(tablesToDrop);
|
|
792
|
+
}
|
|
793
|
+
this.relationCache.delete(rel);
|
|
794
|
+
} else if (changeDocument.operationType == 'rename') {
|
|
795
|
+
const relFrom = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
|
|
796
|
+
const relTo = getMongoRelation(changeDocument.to, this.connections.connectionTag);
|
|
797
|
+
const tablesFrom = await this.getRelations(batch, relFrom, {
|
|
798
|
+
// We're "dropping" this collection, so never snapshot it.
|
|
799
|
+
snapshot: false
|
|
800
|
+
});
|
|
801
|
+
const tablesToDrop = tablesFrom.filter((table) => table.syncAny);
|
|
802
|
+
if (tablesToDrop.length > 0) {
|
|
803
|
+
await batch.drop(tablesToDrop);
|
|
804
|
+
}
|
|
805
|
+
this.relationCache.delete(relFrom);
|
|
806
|
+
// Here we do need to snapshot the new table
|
|
807
|
+
const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
|
|
808
|
+
await this.handleRelation(batch, relTo, {
|
|
809
|
+
// This is a new (renamed) collection, so always snapshot it.
|
|
810
|
+
snapshot: true,
|
|
811
|
+
collectionInfo: collection
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
}
|
|
638
815
|
if (this.abortSignal.aborted) break;
|
|
639
816
|
this.touch();
|
|
640
|
-
|
|
641
|
-
|
|
817
|
+
const batchFilteredCount = sourceBatch.metadata.filteredCount;
|
|
818
|
+
if (batchFilteredCount != null) {
|
|
819
|
+
filteredSinceProgress = (filteredSinceProgress ?? 0) + batchFilteredCount;
|
|
820
|
+
}
|
|
821
|
+
const resumeToken = sourceBatch.getResumeToken();
|
|
822
|
+
if (resumeToken != null) {
|
|
823
|
+
const filteredCount = filteredSinceProgress;
|
|
642
824
|
if (changesSinceProgress == 0) {
|
|
643
|
-
// No retained changes since the previous progress
|
|
825
|
+
// No retained changes since the previous progress boundary: this is either idle or filtered-only traffic.
|
|
644
826
|
// Case 1: No pending barrier and the keepalive interval has elapsed. Advance checkpoints as below,
|
|
645
827
|
// then flush and save the resume token.
|
|
646
828
|
// Case 2: Idle traffic with a recent keepalive. Skip this resume update, whether or not a barrier
|
|
@@ -650,7 +832,7 @@ export class ChangeStream {
|
|
|
650
832
|
// progress still saves its token even during the keepalive interval, so excluded work is resumable.
|
|
651
833
|
const hadRecentKeepalive = performance.now() - lastKeepalive < this.keepaliveIntervalMs;
|
|
652
834
|
if (waitForCheckpointLsn == null && !hadRecentKeepalive) {
|
|
653
|
-
if (filteredCount > 0) {
|
|
835
|
+
if ((filteredCount ?? 0) > 0) {
|
|
654
836
|
// Case 1a: A transaction can span batches: this batch may contain only filtered changes,
|
|
655
837
|
// while retained changes from the same transaction are still unread. They share a timestamp,
|
|
656
838
|
// so publishing a checkpoint here could acknowledge the transaction before its data is saved.
|
|
@@ -666,7 +848,7 @@ export class ChangeStream {
|
|
|
666
848
|
}
|
|
667
849
|
this.touch();
|
|
668
850
|
lastKeepalive = performance.now();
|
|
669
|
-
} else if (filteredCount == 0 && hadRecentKeepalive) {
|
|
851
|
+
} else if ((filteredCount ?? 0) == 0 && hadRecentKeepalive) {
|
|
670
852
|
// Case 2: Only idle progress skips resume persistence. Cases 1 and 3 fall through below.
|
|
671
853
|
continue;
|
|
672
854
|
}
|
|
@@ -681,179 +863,20 @@ export class ChangeStream {
|
|
|
681
863
|
this.lastPersistedResumeTimestamp = timestamp?.getTime() ?? Date.now();
|
|
682
864
|
processingSpan?.end();
|
|
683
865
|
const durationsMicroseconds = outerSpan.end();
|
|
684
|
-
|
|
866
|
+
const filteringDescription = filteredCount == null ? '' : ` and ${filteredCount} filtered events`;
|
|
867
|
+
this.logger.info(`Processed ${changesSinceProgress} changes${filteringDescription}`, {
|
|
685
868
|
count: changesSinceProgress,
|
|
686
|
-
filteredCount,
|
|
869
|
+
...(filteredCount == null ? {} : { filteredCount }),
|
|
687
870
|
bytes: receivedBytes,
|
|
688
871
|
duration: processingSpan?.durationMillis,
|
|
689
872
|
t: durationsMicroseconds
|
|
690
873
|
});
|
|
691
874
|
changesSinceProgress = 0;
|
|
875
|
+
filteredSinceProgress = null;
|
|
692
876
|
receivedBytes = 0;
|
|
693
877
|
outerSpan = tracer.span('batch');
|
|
694
878
|
continue;
|
|
695
879
|
}
|
|
696
|
-
|
|
697
|
-
// The shared reader has already reassembled complete events and normalized their namespaces.
|
|
698
|
-
const changeDocument = item.event;
|
|
699
|
-
changesSinceProgress++;
|
|
700
|
-
const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
|
|
701
|
-
|
|
702
|
-
if (ns?.coll == CHECKPOINTS_COLLECTION) {
|
|
703
|
-
/**
|
|
704
|
-
* Dropping the database does not provide an `invalidate` event.
|
|
705
|
-
* We typically would receive `drop` events for the collection which we
|
|
706
|
-
* would process below.
|
|
707
|
-
*
|
|
708
|
-
* However we don't commit the LSN after collections are dropped.
|
|
709
|
-
* This prevents the `startAfter` or `resumeToken` from advancing past the drop events.
|
|
710
|
-
* The stream also closes after the drop events.
|
|
711
|
-
* This causes an infinite loop of processing the collection drop events.
|
|
712
|
-
*
|
|
713
|
-
* This check here invalidates the change stream if our `_powersync_checkpoints` collection
|
|
714
|
-
* is dropped. This allows for detecting when the DB is dropped.
|
|
715
|
-
*/
|
|
716
|
-
if (changeDocument.operationType == 'drop') {
|
|
717
|
-
throw new ChangeStreamInvalidatedError(
|
|
718
|
-
'Internal collections have been dropped',
|
|
719
|
-
new Error('_powersync_checkpoints collection was dropped')
|
|
720
|
-
);
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
if (
|
|
724
|
-
!(
|
|
725
|
-
changeDocument.operationType == 'insert' ||
|
|
726
|
-
changeDocument.operationType == 'update' ||
|
|
727
|
-
changeDocument.operationType == 'replace'
|
|
728
|
-
)
|
|
729
|
-
) {
|
|
730
|
-
continue;
|
|
731
|
-
}
|
|
732
|
-
|
|
733
|
-
// We handle two types of checkpoint events:
|
|
734
|
-
// 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
|
|
735
|
-
// immediately, regardless of where they were created.
|
|
736
|
-
// 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
|
|
737
|
-
// limiting of commits, so we specifically want to exclude checkpoints from other streams.
|
|
738
|
-
//
|
|
739
|
-
// It may be useful to also throttle commits due to standalone checkpoints in the future.
|
|
740
|
-
// However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
|
|
741
|
-
|
|
742
|
-
const kind = this.checkpointImplementation.event.observe(changeDocument);
|
|
743
|
-
|
|
744
|
-
if (kind == 'foreign') {
|
|
745
|
-
// Another stream's barrier - ignore.
|
|
746
|
-
continue;
|
|
747
|
-
} else if (kind == 'standalone') {
|
|
748
|
-
// Standalone / write checkpoint received.
|
|
749
|
-
// When we are caught up, commit immediately to keep write checkpoint latency low.
|
|
750
|
-
// Once there is already a batch checkpoint pending, or the driver has buffered more
|
|
751
|
-
// change stream events, collapse standalone checkpoints into the normal batch
|
|
752
|
-
// checkpoint flow to avoid commit churn under sustained load.
|
|
753
|
-
const hasBufferedChanges = item.hasBufferedChanges;
|
|
754
|
-
if (hasBufferedChanges && waitForCheckpointLsn == null) {
|
|
755
|
-
// Buffered changes - create a new batch checkpoint to rate limit commits
|
|
756
|
-
using _ = tracer.span('source_checkpoint');
|
|
757
|
-
waitForCheckpointLsn = await this.createBatchCheckpoint();
|
|
758
|
-
continue;
|
|
759
|
-
} else if (waitForCheckpointLsn != null) {
|
|
760
|
-
// Skip this checkpoint - wait for the batch checkpoint.
|
|
761
|
-
continue;
|
|
762
|
-
} else {
|
|
763
|
-
// No buffered changes, and no batch checkpoint pending - commit immediately.
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
// kind == 'own-barrier' falls through to commit.
|
|
767
|
-
|
|
768
|
-
const lsn = this.checkpointImplementation.event.lsn(changeDocument);
|
|
769
|
-
|
|
770
|
-
if (
|
|
771
|
-
waitForCheckpointLsn != null &&
|
|
772
|
-
this.checkpointImplementation.event.resolvesBarrier(waitForCheckpointLsn, changeDocument)
|
|
773
|
-
) {
|
|
774
|
-
waitForCheckpointLsn = null;
|
|
775
|
-
}
|
|
776
|
-
const { checkpointBlocked, checkpointCreated } = await batch.commit(lsn, {
|
|
777
|
-
oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
|
|
778
|
-
});
|
|
779
|
-
|
|
780
|
-
if (!checkpointBlocked || checkpointCreated) {
|
|
781
|
-
this.replicationLag.markCommitted();
|
|
782
|
-
}
|
|
783
|
-
} else if (
|
|
784
|
-
changeDocument.operationType == 'insert' ||
|
|
785
|
-
changeDocument.operationType == 'update' ||
|
|
786
|
-
changeDocument.operationType == 'replace' ||
|
|
787
|
-
changeDocument.operationType == 'delete'
|
|
788
|
-
) {
|
|
789
|
-
if (waitForCheckpointLsn == null) {
|
|
790
|
-
using _ = tracer.span('source_checkpoint');
|
|
791
|
-
waitForCheckpointLsn = await this.createBatchCheckpoint();
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
|
|
795
|
-
const tables = await this.getRelations(batch, rel, {
|
|
796
|
-
// In most cases, we should not need to snapshot this. But if this is the first time we see the collection
|
|
797
|
-
// for whatever reason, then we do need to snapshot it.
|
|
798
|
-
// This may result in some duplicate operations when a collection is created for the first time after
|
|
799
|
-
// sync config was deployed.
|
|
800
|
-
snapshot: true
|
|
801
|
-
});
|
|
802
|
-
const tablesToReplicate = tables.filter((table) => table.syncAny);
|
|
803
|
-
if (tablesToReplicate.length > 0) {
|
|
804
|
-
this.replicationLag.trackUncommittedChange(
|
|
805
|
-
// Standard MongoDB uses clusterTime, unchanged. DocumentDB has no
|
|
806
|
-
// clusterTime, so fall back to wallTime there for the lag metric.
|
|
807
|
-
changeDocument.clusterTime != null
|
|
808
|
-
? timestampToDate(changeDocument.clusterTime)
|
|
809
|
-
: ((changeDocument as any).wallTime ?? null)
|
|
810
|
-
);
|
|
811
|
-
|
|
812
|
-
const transactionKeyValue = transactionKey(changeDocument);
|
|
813
|
-
|
|
814
|
-
if (transactionKeyValue == null || lastTxnKey != transactionKeyValue) {
|
|
815
|
-
// Very crude metric for counting transactions replicated.
|
|
816
|
-
// We ignore operations other than basic CRUD, and ignore changes to _powersync_checkpoints.
|
|
817
|
-
// Individual writes may not have a txnNumber, in which case we count them as separate transactions.
|
|
818
|
-
lastTxnKey = transactionKeyValue;
|
|
819
|
-
transactionsReplicatedMetric.add(1);
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
for (const table of tablesToReplicate) {
|
|
823
|
-
await this.writeChange(batch, table, changeDocument);
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
} else if (changeDocument.operationType == 'drop') {
|
|
827
|
-
const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
|
|
828
|
-
const tables = await this.getRelations(batch, rel, {
|
|
829
|
-
// We're "dropping" this collection, so never snapshot it.
|
|
830
|
-
snapshot: false
|
|
831
|
-
});
|
|
832
|
-
const tablesToDrop = tables.filter((table) => table.syncAny);
|
|
833
|
-
if (tablesToDrop.length > 0) {
|
|
834
|
-
await batch.drop(tablesToDrop);
|
|
835
|
-
}
|
|
836
|
-
this.relationCache.delete(rel);
|
|
837
|
-
} else if (changeDocument.operationType == 'rename') {
|
|
838
|
-
const relFrom = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
|
|
839
|
-
const relTo = getMongoRelation(changeDocument.to, this.connections.connectionTag);
|
|
840
|
-
const tablesFrom = await this.getRelations(batch, relFrom, {
|
|
841
|
-
// We're "dropping" this collection, so never snapshot it.
|
|
842
|
-
snapshot: false
|
|
843
|
-
});
|
|
844
|
-
const tablesToDrop = tablesFrom.filter((table) => table.syncAny);
|
|
845
|
-
if (tablesToDrop.length > 0) {
|
|
846
|
-
await batch.drop(tablesToDrop);
|
|
847
|
-
}
|
|
848
|
-
this.relationCache.delete(relFrom);
|
|
849
|
-
// Here we do need to snapshot the new table
|
|
850
|
-
const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
|
|
851
|
-
await this.handleRelation(batch, relTo, {
|
|
852
|
-
// This is a new (renamed) collection, so always snapshot it.
|
|
853
|
-
snapshot: true,
|
|
854
|
-
collectionInfo: collection
|
|
855
|
-
});
|
|
856
|
-
}
|
|
857
880
|
}
|
|
858
881
|
}
|
|
859
882
|
);
|
|
@@ -70,15 +70,22 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
|
|
|
70
70
|
if (this.isStopped) {
|
|
71
71
|
return;
|
|
72
72
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
73
|
+
let stream: ChangeStream;
|
|
74
|
+
try {
|
|
75
|
+
stream = new ChangeStream({
|
|
76
|
+
abort_signal: this.abortController.signal,
|
|
77
|
+
storage: this.options.storage,
|
|
78
|
+
metrics: this.options.metrics,
|
|
79
|
+
connections: connectionManager,
|
|
80
|
+
keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
|
|
81
|
+
logger: this.logger,
|
|
82
|
+
createReplicationQueryProvider: this.createReplicationQueryProvider
|
|
83
|
+
});
|
|
84
|
+
} catch (error) {
|
|
85
|
+
// Construction can fail before ChangeStream.replicate() reports errors.
|
|
86
|
+
await this.storage.reportError(error);
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
82
89
|
this.lastStream = stream;
|
|
83
90
|
await stream.replicate();
|
|
84
91
|
} finally {
|
|
@@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { SourceTable } from '@powersync/service-core';
|
|
3
3
|
import { HydratedSyncConfig } from '@powersync/service-sync-rules';
|
|
4
4
|
import { MongoManager } from './MongoManager.js';
|
|
5
|
-
import {
|
|
5
|
+
import type { MongoReplicationStreamBatch, MongoReplicationStreamContext } from './MongoReplicationStreamTypes.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Sync config and source namespace information used to construct a provider for a replication attempt.
|
|
@@ -35,17 +35,15 @@ export interface MongoReplicationQueryProvider {
|
|
|
35
35
|
validateSource?(options: { connectionManager: MongoManager; isDocumentDb: boolean }): Promise<void>;
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
|
-
* Call context.open() with any additional stages/pre-image options
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* Each call opens a separate iterator, also used for snapshot barriers and resume validation. Keep
|
|
46
|
-
* per-iterator state separate and close the underlying iterator when the wrapper exits.
|
|
38
|
+
* Call context.open() with any additional stages/pre-image options. For filtering, pass its result
|
|
39
|
+
* to filterMongoReplicationStream({ batches, filter }) with a MongoReplicationFilter callback.
|
|
40
|
+
* The helper preserves checkpoints, DDL, batch boundaries, buffered-change flags and safe progress,
|
|
41
|
+
* counts excluded events, and closes the source iterator on failure or early exit.
|
|
42
|
+
* The default provider returns the opener's batches directly.
|
|
43
|
+
* Each invocation is independent and also serves snapshot barriers and resume validation; keep
|
|
44
|
+
* filter state local to that invocation. Snapshot selection must agree with streaming decisions.
|
|
47
45
|
*/
|
|
48
|
-
openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<
|
|
46
|
+
openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<MongoReplicationStreamBatch>;
|
|
49
47
|
|
|
50
48
|
/**
|
|
51
49
|
* Returns an additional MongoDB find predicate for a physical source table.
|