@powersync/service-module-mongodb 0.0.0-dev-20261005120023 → 0.0.0-dev-20261015071128

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +38 -7
  2. package/benchmarks/env.ts +11 -0
  3. package/benchmarks/replication_branch_throughput.benchmark.ts +240 -0
  4. package/dist/api/MongoRouteAPIAdapter.d.ts +6 -0
  5. package/dist/api/MongoRouteAPIAdapter.js +21 -2
  6. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  7. package/dist/module/MongoModule.js +1 -0
  8. package/dist/module/MongoModule.js.map +1 -1
  9. package/dist/replication/ChangeStream.d.ts +3 -1
  10. package/dist/replication/ChangeStream.js +173 -150
  11. package/dist/replication/ChangeStream.js.map +1 -1
  12. package/dist/replication/ChangeStreamReplicationJob.js +17 -9
  13. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  14. package/dist/replication/MongoReplicationQueryProvider.d.ts +9 -11
  15. package/dist/replication/MongoReplicationQueryProvider.js.map +1 -1
  16. package/dist/replication/MongoReplicationStream.d.ts +37 -48
  17. package/dist/replication/MongoReplicationStream.js +157 -30
  18. package/dist/replication/MongoReplicationStream.js.map +1 -1
  19. package/dist/replication/MongoReplicationStreamTypes.d.ts +157 -0
  20. package/dist/replication/MongoReplicationStreamTypes.js +19 -0
  21. package/dist/replication/MongoReplicationStreamTypes.js.map +1 -0
  22. package/dist/replication/MongoSnapshotter.js +45 -25
  23. package/dist/replication/MongoSnapshotter.js.map +1 -1
  24. package/dist/replication/replication-index.d.ts +1 -0
  25. package/dist/replication/replication-index.js +1 -0
  26. package/dist/replication/replication-index.js.map +1 -1
  27. package/dist/test-utils/ChangeStreamTestContext.js +2 -3
  28. package/dist/test-utils/ChangeStreamTestContext.js.map +1 -1
  29. package/package.json +11 -10
  30. package/src/api/MongoRouteAPIAdapter.ts +26 -2
  31. package/src/module/MongoModule.ts +1 -0
  32. package/src/replication/ChangeStream.ts +197 -174
  33. package/src/replication/ChangeStreamReplicationJob.ts +16 -9
  34. package/src/replication/MongoReplicationQueryProvider.ts +9 -11
  35. package/src/replication/MongoReplicationStream.ts +196 -83
  36. package/src/replication/MongoReplicationStreamTypes.ts +169 -0
  37. package/src/replication/MongoSnapshotter.ts +49 -28
  38. package/src/replication/replication-index.ts +1 -0
  39. package/src/test-utils/ChangeStreamTestContext.ts +2 -2
  40. package/test/src/change_event_memory.test.ts +33 -0
  41. package/test/src/change_stream_abort.test.ts +110 -0
  42. package/test/src/filtering_example.test.ts +65 -0
  43. package/test/src/filtering_lifecycle.test.ts +101 -0
  44. package/test/src/query_provider_availability.test.ts +77 -0
  45. package/test/src/query_provider_diagnostics.test.ts +55 -0
  46. package/test/src/replication_stream.test.ts +411 -34
  47. package/test/src/stream_progress.test.ts +62 -44
  48. package/test/tsconfig.json +2 -2
  49. package/tsconfig.tsbuildinfo +1 -1
  50. package/vitest.benchmark.config.ts +15 -0
@@ -18,7 +18,7 @@ import {
18
18
  SourceTable,
19
19
  storage
20
20
  } from '@powersync/service-core';
21
- import { HydratedSyncConfig } from '@powersync/service-sync-rules';
21
+ import { hasMongoFilterExpressions, HydratedSyncConfig } from '@powersync/service-sync-rules';
22
22
  import { ReplicationMetric } from '@powersync/service-types';
23
23
  import { performance } from 'node:perf_hooks';
24
24
  import { PostImagesOption } from '../types/types.js';
@@ -32,14 +32,15 @@ import {
32
32
  MongoReplicationQueryProvider,
33
33
  MongoReplicationQueryProviderFactory
34
34
  } from './MongoReplicationQueryProvider.js';
35
- import { MongoReplicationStreamItem, openMongoReplicationStream } from './MongoReplicationStream.js';
35
+ import { openMongoReplicationStream } from './MongoReplicationStream.js';
36
+ import type { MongoReplicationStreamBatch } from './MongoReplicationStreamTypes.js';
36
37
  import { MongoSnapshotter, MongoSnapshotterHooks } from './MongoSnapshotter.js';
37
38
  import { ChangeStreamBatch, ProjectedChangeStreamDocument } from './RawChangeStream.js';
38
39
  import { CHECKPOINTS_COLLECTION, detectDocumentDb, timestampToDate } from './replication-utils.js';
39
40
  import { DirectSourceRowConverter, SourceRowConverter } from './SourceRowConverter.js';
40
41
  export interface ChangeStreamOptions {
41
- connections: MongoManager;
42
42
  storage: storage.SyncRulesBucketStorage;
43
+ connections: MongoManager;
43
44
  metrics: MetricsEngine;
44
45
  abort_signal: AbortSignal;
45
46
  /**
@@ -124,6 +125,7 @@ export class ChangeStream {
124
125
 
125
126
  private readonly sourceRowConverter: SourceRowConverter;
126
127
  private readonly queryProvider: MongoReplicationQueryProvider;
128
+ private readonly hasQueryProviderFactory: boolean;
127
129
 
128
130
  private keepaliveIntervalMs: number;
129
131
 
@@ -136,7 +138,7 @@ export class ChangeStream {
136
138
  private lastPersistedResumeTimestamp = 0;
137
139
  private lastBatchCheckpoint = 0;
138
140
 
139
- constructor(options: ChangeStreamOptions) {
141
+ constructor(private readonly options: ChangeStreamOptions) {
140
142
  this.storage = options.storage;
141
143
  this.metrics = options.metrics;
142
144
  this.group_id = options.storage.replicationStreamId;
@@ -156,6 +158,7 @@ export class ChangeStream {
156
158
  this.sync_rules = options.storage.getParsedSyncRules({
157
159
  defaultSchema: this.defaultDb.databaseName
158
160
  });
161
+ this.hasQueryProviderFactory = options.createReplicationQueryProvider != null;
159
162
  this.queryProvider =
160
163
  options.createReplicationQueryProvider?.({
161
164
  syncConfig: this.sync_rules,
@@ -501,6 +504,17 @@ export class ChangeStream {
501
504
  }
502
505
 
503
506
  private async initReplication() {
507
+ // A previously deployed config may outlive the module that provided pre-filtering.
508
+ // Throw inside replicate() so the failure is persisted in replication diagnostics.
509
+ if (
510
+ !this.hasQueryProviderFactory &&
511
+ hasMongoFilterExpressions(this.sync_rules.sourceTableConfig, this.connections.connectionTag)
512
+ ) {
513
+ throw new ServiceError(
514
+ ErrorCode.PSYNC_S1348,
515
+ 'This sync config requires MongoDB pre-filtering, but no query provider factory is registered. Restore the required module, or remove the filters and deploy a new sync config to restart replication.'
516
+ );
517
+ }
504
518
  const result = await this.snapshotter.checkSlot();
505
519
  await this.snapshotter.setupCheckpointsCollection();
506
520
  if (result.needsInitialSync) {
@@ -544,7 +558,7 @@ export class ChangeStream {
544
558
  signal?: AbortSignal;
545
559
  tracer?: PerformanceTracer<'changestream'>;
546
560
  onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
547
- }): AsyncIterableIterator<MongoReplicationStreamItem> {
561
+ }): AsyncIterableIterator<MongoReplicationStreamBatch> {
548
562
  return openMongoReplicationStream({
549
563
  db: this.defaultDb,
550
564
  queryProvider: this.queryProvider,
@@ -611,6 +625,7 @@ export class ChangeStream {
611
625
  let processingSpan: ReturnType<typeof tracer.span> | undefined;
612
626
  let receivedBytes = 0;
613
627
  let changesSinceProgress = 0;
628
+ let filteredSinceProgress: number | null = null;
614
629
  const stream = this.openChangeStream({
615
630
  lsn: resumeFromLsn,
616
631
  filters,
@@ -634,13 +649,180 @@ export class ChangeStream {
634
649
  let lastKeepalive = performance.now();
635
650
  let lastTxnKey: string | null = null;
636
651
 
637
- for await (const item of stream) {
652
+ streamLoop: for await (const sourceBatch of stream) {
653
+ if (this.abortSignal.aborted) break;
654
+ for (const { event: changeDocument, hasBufferedChanges } of sourceBatch.events) {
655
+ if (this.abortSignal.aborted) break streamLoop;
656
+ this.touch();
657
+ changesSinceProgress++;
658
+ const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
659
+
660
+ if (ns?.coll == CHECKPOINTS_COLLECTION) {
661
+ /**
662
+ * Dropping the database does not provide an `invalidate` event.
663
+ * We typically would receive `drop` events for the collection which we
664
+ * would process below.
665
+ *
666
+ * However we don't commit the LSN after collections are dropped.
667
+ * This prevents the `startAfter` or `resumeToken` from advancing past the drop events.
668
+ * The stream also closes after the drop events.
669
+ * This causes an infinite loop of processing the collection drop events.
670
+ *
671
+ * This check here invalidates the change stream if our `_powersync_checkpoints` collection
672
+ * is dropped. This allows for detecting when the DB is dropped.
673
+ */
674
+ if (changeDocument.operationType == 'drop') {
675
+ throw new ChangeStreamInvalidatedError(
676
+ 'Internal collections have been dropped',
677
+ new Error('_powersync_checkpoints collection was dropped')
678
+ );
679
+ }
680
+
681
+ if (
682
+ !(
683
+ changeDocument.operationType == 'insert' ||
684
+ changeDocument.operationType == 'update' ||
685
+ changeDocument.operationType == 'replace'
686
+ )
687
+ ) {
688
+ continue;
689
+ }
690
+
691
+ // We handle two types of checkpoint events:
692
+ // 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
693
+ // immediately, regardless of where they were created.
694
+ // 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
695
+ // limiting of commits, so we specifically want to exclude checkpoints from other streams.
696
+ //
697
+ // It may be useful to also throttle commits due to standalone checkpoints in the future.
698
+ // However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
699
+
700
+ const kind = this.checkpointImplementation.event.observe(changeDocument);
701
+
702
+ if (kind == 'foreign') {
703
+ // Another stream's barrier - ignore.
704
+ continue;
705
+ } else if (kind == 'standalone') {
706
+ // Standalone / write checkpoint received.
707
+ // When we are caught up, commit immediately to keep write checkpoint latency low.
708
+ // Once there is already a batch checkpoint pending, or the driver has buffered more
709
+ // change stream events, collapse standalone checkpoints into the normal batch
710
+ // checkpoint flow to avoid commit churn under sustained load.
711
+ if (hasBufferedChanges && waitForCheckpointLsn == null) {
712
+ // Buffered changes - create a new batch checkpoint to rate limit commits
713
+ using _ = tracer.span('source_checkpoint');
714
+ waitForCheckpointLsn = await this.createBatchCheckpoint();
715
+ continue;
716
+ } else if (waitForCheckpointLsn != null) {
717
+ // Skip this checkpoint - wait for the batch checkpoint.
718
+ continue;
719
+ } else {
720
+ // No buffered changes, and no batch checkpoint pending - commit immediately.
721
+ }
722
+ }
723
+ // kind == 'own-barrier' falls through to commit.
724
+
725
+ const lsn = this.checkpointImplementation.event.lsn(changeDocument);
726
+
727
+ if (
728
+ waitForCheckpointLsn != null &&
729
+ this.checkpointImplementation.event.resolvesBarrier(waitForCheckpointLsn, changeDocument)
730
+ ) {
731
+ waitForCheckpointLsn = null;
732
+ }
733
+ const { checkpointBlocked, checkpointCreated } = await batch.commit(lsn, {
734
+ oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
735
+ });
736
+
737
+ if (!checkpointBlocked || checkpointCreated) {
738
+ this.replicationLag.markCommitted();
739
+ }
740
+ } else if (
741
+ changeDocument.operationType == 'insert' ||
742
+ changeDocument.operationType == 'update' ||
743
+ changeDocument.operationType == 'replace' ||
744
+ changeDocument.operationType == 'delete'
745
+ ) {
746
+ if (waitForCheckpointLsn == null) {
747
+ using _ = tracer.span('source_checkpoint');
748
+ waitForCheckpointLsn = await this.createBatchCheckpoint();
749
+ }
750
+
751
+ const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
752
+ const tables = await this.getRelations(batch, rel, {
753
+ // In most cases, we should not need to snapshot this. But if this is the first time we see the collection
754
+ // for whatever reason, then we do need to snapshot it.
755
+ // This may result in some duplicate operations when a collection is created for the first time after
756
+ // sync config was deployed.
757
+ snapshot: true
758
+ });
759
+ const tablesToReplicate = tables.filter((table) => table.syncAny);
760
+ if (tablesToReplicate.length > 0) {
761
+ this.replicationLag.trackUncommittedChange(
762
+ // Standard MongoDB uses clusterTime, unchanged. DocumentDB has no
763
+ // clusterTime, so fall back to wallTime there for the lag metric.
764
+ changeDocument.clusterTime != null
765
+ ? timestampToDate(changeDocument.clusterTime)
766
+ : ((changeDocument as any).wallTime ?? null)
767
+ );
768
+
769
+ const transactionKeyValue = transactionKey(changeDocument);
770
+
771
+ if (transactionKeyValue == null || lastTxnKey != transactionKeyValue) {
772
+ // Very crude metric for counting transactions replicated.
773
+ // We ignore operations other than basic CRUD, and ignore changes to _powersync_checkpoints.
774
+ // Individual writes may not have a txnNumber, in which case we count them as separate transactions.
775
+ lastTxnKey = transactionKeyValue;
776
+ transactionsReplicatedMetric.add(1);
777
+ }
778
+
779
+ for (const table of tablesToReplicate) {
780
+ await this.writeChange(batch, table, changeDocument);
781
+ }
782
+ }
783
+ } else if (changeDocument.operationType == 'drop') {
784
+ const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
785
+ const tables = await this.getRelations(batch, rel, {
786
+ // We're "dropping" this collection, so never snapshot it.
787
+ snapshot: false
788
+ });
789
+ const tablesToDrop = tables.filter((table) => table.syncAny);
790
+ if (tablesToDrop.length > 0) {
791
+ await batch.drop(tablesToDrop);
792
+ }
793
+ this.relationCache.delete(rel);
794
+ } else if (changeDocument.operationType == 'rename') {
795
+ const relFrom = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
796
+ const relTo = getMongoRelation(changeDocument.to, this.connections.connectionTag);
797
+ const tablesFrom = await this.getRelations(batch, relFrom, {
798
+ // We're "dropping" this collection, so never snapshot it.
799
+ snapshot: false
800
+ });
801
+ const tablesToDrop = tablesFrom.filter((table) => table.syncAny);
802
+ if (tablesToDrop.length > 0) {
803
+ await batch.drop(tablesToDrop);
804
+ }
805
+ this.relationCache.delete(relFrom);
806
+ // Here we do need to snapshot the new table
807
+ const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
808
+ await this.handleRelation(batch, relTo, {
809
+ // This is a new (renamed) collection, so always snapshot it.
810
+ snapshot: true,
811
+ collectionInfo: collection
812
+ });
813
+ }
814
+ }
638
815
  if (this.abortSignal.aborted) break;
639
816
  this.touch();
640
- if (item.type == 'progress') {
641
- const { resumeToken, filteredCount } = item;
817
+ const batchFilteredCount = sourceBatch.metadata.filteredCount;
818
+ if (batchFilteredCount != null) {
819
+ filteredSinceProgress = (filteredSinceProgress ?? 0) + batchFilteredCount;
820
+ }
821
+ const resumeToken = sourceBatch.getResumeToken();
822
+ if (resumeToken != null) {
823
+ const filteredCount = filteredSinceProgress;
642
824
  if (changesSinceProgress == 0) {
643
- // No retained changes since the previous progress item: this is either idle or filtered-only traffic.
825
+ // No retained changes since the previous progress boundary: this is either idle or filtered-only traffic.
644
826
  // Case 1: No pending barrier and the keepalive interval has elapsed. Advance checkpoints as below,
645
827
  // then flush and save the resume token.
646
828
  // Case 2: Idle traffic with a recent keepalive. Skip this resume update, whether or not a barrier
@@ -650,7 +832,7 @@ export class ChangeStream {
650
832
  // progress still saves its token even during the keepalive interval, so excluded work is resumable.
651
833
  const hadRecentKeepalive = performance.now() - lastKeepalive < this.keepaliveIntervalMs;
652
834
  if (waitForCheckpointLsn == null && !hadRecentKeepalive) {
653
- if (filteredCount > 0) {
835
+ if ((filteredCount ?? 0) > 0) {
654
836
  // Case 1a: A transaction can span batches: this batch may contain only filtered changes,
655
837
  // while retained changes from the same transaction are still unread. They share a timestamp,
656
838
  // so publishing a checkpoint here could acknowledge the transaction before its data is saved.
@@ -666,7 +848,7 @@ export class ChangeStream {
666
848
  }
667
849
  this.touch();
668
850
  lastKeepalive = performance.now();
669
- } else if (filteredCount == 0 && hadRecentKeepalive) {
851
+ } else if ((filteredCount ?? 0) == 0 && hadRecentKeepalive) {
670
852
  // Case 2: Only idle progress skips resume persistence. Cases 1 and 3 fall through below.
671
853
  continue;
672
854
  }
@@ -681,179 +863,20 @@ export class ChangeStream {
681
863
  this.lastPersistedResumeTimestamp = timestamp?.getTime() ?? Date.now();
682
864
  processingSpan?.end();
683
865
  const durationsMicroseconds = outerSpan.end();
684
- this.logger.info(`Processed ${changesSinceProgress} changes and ${filteredCount} filtered events`, {
866
+ const filteringDescription = filteredCount == null ? '' : ` and ${filteredCount} filtered events`;
867
+ this.logger.info(`Processed ${changesSinceProgress} changes${filteringDescription}`, {
685
868
  count: changesSinceProgress,
686
- filteredCount,
869
+ ...(filteredCount == null ? {} : { filteredCount }),
687
870
  bytes: receivedBytes,
688
871
  duration: processingSpan?.durationMillis,
689
872
  t: durationsMicroseconds
690
873
  });
691
874
  changesSinceProgress = 0;
875
+ filteredSinceProgress = null;
692
876
  receivedBytes = 0;
693
877
  outerSpan = tracer.span('batch');
694
878
  continue;
695
879
  }
696
-
697
- // The shared reader has already reassembled complete events and normalized their namespaces.
698
- const changeDocument = item.event;
699
- changesSinceProgress++;
700
- const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
701
-
702
- if (ns?.coll == CHECKPOINTS_COLLECTION) {
703
- /**
704
- * Dropping the database does not provide an `invalidate` event.
705
- * We typically would receive `drop` events for the collection which we
706
- * would process below.
707
- *
708
- * However we don't commit the LSN after collections are dropped.
709
- * This prevents the `startAfter` or `resumeToken` from advancing past the drop events.
710
- * The stream also closes after the drop events.
711
- * This causes an infinite loop of processing the collection drop events.
712
- *
713
- * This check here invalidates the change stream if our `_powersync_checkpoints` collection
714
- * is dropped. This allows for detecting when the DB is dropped.
715
- */
716
- if (changeDocument.operationType == 'drop') {
717
- throw new ChangeStreamInvalidatedError(
718
- 'Internal collections have been dropped',
719
- new Error('_powersync_checkpoints collection was dropped')
720
- );
721
- }
722
-
723
- if (
724
- !(
725
- changeDocument.operationType == 'insert' ||
726
- changeDocument.operationType == 'update' ||
727
- changeDocument.operationType == 'replace'
728
- )
729
- ) {
730
- continue;
731
- }
732
-
733
- // We handle two types of checkpoint events:
734
- // 1. "Standalone" checkpoints, typically write checkpoints. We want to process these
735
- // immediately, regardless of where they were created.
736
- // 2. "Batch" checkpoints for the current stream. This is used as a form of dynamic rate
737
- // limiting of commits, so we specifically want to exclude checkpoints from other streams.
738
- //
739
- // It may be useful to also throttle commits due to standalone checkpoints in the future.
740
- // However, these typically have a much lower rate than batch checkpoints, so we don't do that for now.
741
-
742
- const kind = this.checkpointImplementation.event.observe(changeDocument);
743
-
744
- if (kind == 'foreign') {
745
- // Another stream's barrier - ignore.
746
- continue;
747
- } else if (kind == 'standalone') {
748
- // Standalone / write checkpoint received.
749
- // When we are caught up, commit immediately to keep write checkpoint latency low.
750
- // Once there is already a batch checkpoint pending, or the driver has buffered more
751
- // change stream events, collapse standalone checkpoints into the normal batch
752
- // checkpoint flow to avoid commit churn under sustained load.
753
- const hasBufferedChanges = item.hasBufferedChanges;
754
- if (hasBufferedChanges && waitForCheckpointLsn == null) {
755
- // Buffered changes - create a new batch checkpoint to rate limit commits
756
- using _ = tracer.span('source_checkpoint');
757
- waitForCheckpointLsn = await this.createBatchCheckpoint();
758
- continue;
759
- } else if (waitForCheckpointLsn != null) {
760
- // Skip this checkpoint - wait for the batch checkpoint.
761
- continue;
762
- } else {
763
- // No buffered changes, and no batch checkpoint pending - commit immediately.
764
- }
765
- }
766
- // kind == 'own-barrier' falls through to commit.
767
-
768
- const lsn = this.checkpointImplementation.event.lsn(changeDocument);
769
-
770
- if (
771
- waitForCheckpointLsn != null &&
772
- this.checkpointImplementation.event.resolvesBarrier(waitForCheckpointLsn, changeDocument)
773
- ) {
774
- waitForCheckpointLsn = null;
775
- }
776
- const { checkpointBlocked, checkpointCreated } = await batch.commit(lsn, {
777
- oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
778
- });
779
-
780
- if (!checkpointBlocked || checkpointCreated) {
781
- this.replicationLag.markCommitted();
782
- }
783
- } else if (
784
- changeDocument.operationType == 'insert' ||
785
- changeDocument.operationType == 'update' ||
786
- changeDocument.operationType == 'replace' ||
787
- changeDocument.operationType == 'delete'
788
- ) {
789
- if (waitForCheckpointLsn == null) {
790
- using _ = tracer.span('source_checkpoint');
791
- waitForCheckpointLsn = await this.createBatchCheckpoint();
792
- }
793
-
794
- const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
795
- const tables = await this.getRelations(batch, rel, {
796
- // In most cases, we should not need to snapshot this. But if this is the first time we see the collection
797
- // for whatever reason, then we do need to snapshot it.
798
- // This may result in some duplicate operations when a collection is created for the first time after
799
- // sync config was deployed.
800
- snapshot: true
801
- });
802
- const tablesToReplicate = tables.filter((table) => table.syncAny);
803
- if (tablesToReplicate.length > 0) {
804
- this.replicationLag.trackUncommittedChange(
805
- // Standard MongoDB uses clusterTime, unchanged. DocumentDB has no
806
- // clusterTime, so fall back to wallTime there for the lag metric.
807
- changeDocument.clusterTime != null
808
- ? timestampToDate(changeDocument.clusterTime)
809
- : ((changeDocument as any).wallTime ?? null)
810
- );
811
-
812
- const transactionKeyValue = transactionKey(changeDocument);
813
-
814
- if (transactionKeyValue == null || lastTxnKey != transactionKeyValue) {
815
- // Very crude metric for counting transactions replicated.
816
- // We ignore operations other than basic CRUD, and ignore changes to _powersync_checkpoints.
817
- // Individual writes may not have a txnNumber, in which case we count them as separate transactions.
818
- lastTxnKey = transactionKeyValue;
819
- transactionsReplicatedMetric.add(1);
820
- }
821
-
822
- for (const table of tablesToReplicate) {
823
- await this.writeChange(batch, table, changeDocument);
824
- }
825
- }
826
- } else if (changeDocument.operationType == 'drop') {
827
- const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
828
- const tables = await this.getRelations(batch, rel, {
829
- // We're "dropping" this collection, so never snapshot it.
830
- snapshot: false
831
- });
832
- const tablesToDrop = tables.filter((table) => table.syncAny);
833
- if (tablesToDrop.length > 0) {
834
- await batch.drop(tablesToDrop);
835
- }
836
- this.relationCache.delete(rel);
837
- } else if (changeDocument.operationType == 'rename') {
838
- const relFrom = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
839
- const relTo = getMongoRelation(changeDocument.to, this.connections.connectionTag);
840
- const tablesFrom = await this.getRelations(batch, relFrom, {
841
- // We're "dropping" this collection, so never snapshot it.
842
- snapshot: false
843
- });
844
- const tablesToDrop = tablesFrom.filter((table) => table.syncAny);
845
- if (tablesToDrop.length > 0) {
846
- await batch.drop(tablesToDrop);
847
- }
848
- this.relationCache.delete(relFrom);
849
- // Here we do need to snapshot the new table
850
- const collection = await this.getCollectionInfo(relTo.schema, relTo.name);
851
- await this.handleRelation(batch, relTo, {
852
- // This is a new (renamed) collection, so always snapshot it.
853
- snapshot: true,
854
- collectionInfo: collection
855
- });
856
- }
857
880
  }
858
881
  }
859
882
  );
@@ -70,15 +70,22 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
70
70
  if (this.isStopped) {
71
71
  return;
72
72
  }
73
- const stream = new ChangeStream({
74
- abort_signal: this.abortController.signal,
75
- storage: this.options.storage,
76
- metrics: this.options.metrics,
77
- connections: connectionManager,
78
- keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
79
- logger: this.logger,
80
- createReplicationQueryProvider: this.createReplicationQueryProvider
81
- });
73
+ let stream: ChangeStream;
74
+ try {
75
+ stream = new ChangeStream({
76
+ abort_signal: this.abortController.signal,
77
+ storage: this.options.storage,
78
+ metrics: this.options.metrics,
79
+ connections: connectionManager,
80
+ keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
81
+ logger: this.logger,
82
+ createReplicationQueryProvider: this.createReplicationQueryProvider
83
+ });
84
+ } catch (error) {
85
+ // Construction can fail before ChangeStream.replicate() reports errors.
86
+ await this.storage.reportError(error);
87
+ throw error;
88
+ }
82
89
  this.lastStream = stream;
83
90
  await stream.replicate();
84
91
  } finally {
@@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { SourceTable } from '@powersync/service-core';
3
3
  import { HydratedSyncConfig } from '@powersync/service-sync-rules';
4
4
  import { MongoManager } from './MongoManager.js';
5
- import { MongoReplicationStreamContext, MongoReplicationStreamItem } from './MongoReplicationStream.js';
5
+ import type { MongoReplicationStreamBatch, MongoReplicationStreamContext } from './MongoReplicationStreamTypes.js';
6
6
 
7
7
  /**
8
8
  * Sync config and source namespace information used to construct a provider for a replication attempt.
@@ -35,17 +35,15 @@ export interface MongoReplicationQueryProvider {
35
35
  validateSource?(options: { connectionManager: MongoManager; isDocumentDb: boolean }): Promise<void>;
36
36
 
37
37
  /**
38
- * Call context.open() with any additional stages/pre-image options, then optionally wrap its iterator.
39
- * Unlike pipeline stages alone, the wrapper can exclude marked events while still forwarding progress.
40
- * Yield changes and safe progress, including when all data events are excluded.
41
- * Count excluded complete events in the next progress item's filteredCount, resetting it after each
42
- * progress item. Preserve the reader's safe resume token rather than advancing past buffered changes.
43
- * An exit from a filtered set may be represented as a synthetic delete retaining its original key and token.
44
- * Checkpoint/DDL events must remain ordered and intact. Progress alone must never stand in for a checkpoint.
45
- * Each call opens a separate iterator, also used for snapshot barriers and resume validation. Keep
46
- * per-iterator state separate and close the underlying iterator when the wrapper exits.
38
+ * Call context.open() with any additional stages/pre-image options. For filtering, pass its result
39
+ * to filterMongoReplicationStream({ batches, filter }) with a MongoReplicationFilter callback.
40
+ * The helper preserves checkpoints, DDL, batch boundaries, buffered-change flags and safe progress,
41
+ * counts excluded events, and closes the source iterator on failure or early exit.
42
+ * The default provider returns the opener's batches directly.
43
+ * Each invocation is independent and also serves snapshot barriers and resume validation; keep
44
+ * filter state local to that invocation. Snapshot selection must agree with streaming decisions.
47
45
  */
48
- openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<MongoReplicationStreamItem>;
46
+ openChangeStream(context: MongoReplicationStreamContext): AsyncIterableIterator<MongoReplicationStreamBatch>;
49
47
 
50
48
  /**
51
49
  * Returns an additional MongoDB find predicate for a physical source table.