@powersync/service-module-mongodb 0.0.0-dev-20261007110800 → 0.0.0-dev-20261015132441

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +14 -11
  2. package/benchmarks/env.ts +11 -0
  3. package/benchmarks/replication_branch_throughput.benchmark.ts +240 -0
  4. package/dist/api/MongoRouteAPIAdapter.d.ts +5 -0
  5. package/dist/api/MongoRouteAPIAdapter.js +11 -4
  6. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  7. package/dist/module/MongoModule.js +1 -0
  8. package/dist/module/MongoModule.js.map +1 -1
  9. package/dist/replication/ChangeStream.d.ts +2 -1
  10. package/dist/replication/ChangeStream.js +164 -150
  11. package/dist/replication/ChangeStream.js.map +1 -1
  12. package/dist/replication/ChangeStreamReplicationJob.js +17 -9
  13. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  14. package/dist/replication/MongoReplicationQueryProvider.d.ts +9 -11
  15. package/dist/replication/MongoReplicationQueryProvider.js.map +1 -1
  16. package/dist/replication/MongoReplicationStream.d.ts +37 -48
  17. package/dist/replication/MongoReplicationStream.js +157 -30
  18. package/dist/replication/MongoReplicationStream.js.map +1 -1
  19. package/dist/replication/MongoReplicationStreamTypes.d.ts +157 -0
  20. package/dist/replication/MongoReplicationStreamTypes.js +19 -0
  21. package/dist/replication/MongoReplicationStreamTypes.js.map +1 -0
  22. package/dist/replication/MongoSnapshotter.js +59 -30
  23. package/dist/replication/MongoSnapshotter.js.map +1 -1
  24. package/dist/replication/estimateFilteredSnapshotCount.d.ts +12 -0
  25. package/dist/replication/estimateFilteredSnapshotCount.js +79 -0
  26. package/dist/replication/estimateFilteredSnapshotCount.js.map +1 -0
  27. package/dist/replication/replication-index.d.ts +1 -0
  28. package/dist/replication/replication-index.js +1 -0
  29. package/dist/replication/replication-index.js.map +1 -1
  30. package/package.json +11 -10
  31. package/src/api/MongoRouteAPIAdapter.ts +12 -5
  32. package/src/module/MongoModule.ts +1 -0
  33. package/src/replication/ChangeStream.ts +183 -173
  34. package/src/replication/ChangeStreamReplicationJob.ts +16 -9
  35. package/src/replication/MongoReplicationQueryProvider.ts +9 -11
  36. package/src/replication/MongoReplicationStream.ts +196 -83
  37. package/src/replication/MongoReplicationStreamTypes.ts +169 -0
  38. package/src/replication/MongoSnapshotter.ts +64 -33
  39. package/src/replication/estimateFilteredSnapshotCount.ts +93 -0
  40. package/src/replication/replication-index.ts +1 -0
  41. package/test/src/change_event_memory.test.ts +33 -0
  42. package/test/src/change_stream_abort.test.ts +110 -0
  43. package/test/src/filtering_example.test.ts +65 -0
  44. package/test/src/filtering_lifecycle.test.ts +101 -0
  45. package/test/src/query_provider_availability.test.ts +1 -0
  46. package/test/src/query_provider_diagnostics.test.ts +55 -0
  47. package/test/src/replication_stream.test.ts +411 -34
  48. package/test/src/snapshot_count.test.ts +146 -0
  49. package/test/src/snapshot_count_integration.test.ts +70 -0
  50. package/test/src/snapshot_query_provider.test.ts +21 -2
  51. package/test/src/stream_progress.test.ts +62 -44
  52. package/test/tsconfig.json +2 -2
  53. package/tsconfig.tsbuildinfo +1 -1
  54. package/vitest.benchmark.config.ts +15 -0
@@ -1,50 +1,31 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { Logger } from '@powersync/lib-services-framework';
3
2
  import { MongoReplicationQueryProvider } from './MongoReplicationQueryProvider.js';
3
+ import { MongoReplicationFilterOptions, MongoReplicationParserOptions, MongoReplicationStreamBatch, MongoReplicationStreamBatchMetadata, MongoReplicationStreamChange, RawMongoReplicationStreamBatch } from './MongoReplicationStreamTypes.js';
4
4
  import { ChangeStreamBatch, ProjectedChangeStreamDocument, RawChangeStreamOptions } from './RawChangeStream.js';
5
5
  /**
6
- * Ordered adapter output. Progress advances source recovery, not the client-visible checkpoint.
6
+ * Wraps changes with a single-use iterator. Caches the token and metadata after successful exhaustion.
7
7
  */
8
- export type MongoReplicationStreamItem = {
9
- type: 'change';
10
- event: ProjectedChangeStreamDocument;
8
+ export declare function createMongoReplicationStreamBatch({ events, resolveResumeToken, resolveMetadata }: {
11
9
  /**
12
- * More envelopes remain in the received batch, including ones an adapter may exclude.
10
+ * Complete changes, optionally produced by a lazy parsing or filtering generator.
13
11
  */
14
- hasBufferedChanges: boolean;
15
- } | {
16
- type: 'progress';
12
+ events: Iterable<MongoReplicationStreamChange>;
17
13
  /**
18
- * Covers only complete events already delivered or intentionally excluded by the adapter.
14
+ * Compute a safe resume token once after successful exhaustion, using the final parser state.
19
15
  */
20
- resumeToken: mongo.ResumeToken;
16
+ resolveResumeToken: () => mongo.ResumeToken | undefined;
21
17
  /**
22
- * Excluded complete events since the previous progress item; reset after every boundary.
18
+ * Compute this batch's final statistics after exhaustion. Defaults to no filtering.
23
19
  */
24
- filteredCount: number;
25
- };
26
- export interface MongoReplicationStreamOptions {
27
- /**
28
- * Inserted after namespace selection and before the final large-event split stage.
29
- */
30
- pipelineStages?: mongo.Document[];
31
- /**
32
- * Request pre-images without overriding the reader's post-image, resume or scope options.
33
- */
34
- imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
35
- }
36
- export interface MongoReplicationStreamContext {
37
- /**
38
- * Open the source reader with the current position, namespace, deadlines and cancellation already bound.
39
- * The reader returns parsed, reassembled changes followed by safe progress boundaries. A provider can
40
- * add stages here and wrap the result to discard marked events or turn filter exits into deletes.
41
- * Supplying this callback keeps cursor setup shared with snapshot barriers and resume validation.
42
- *
43
- * The default provider simply returns open({}). Wrappers must preserve event/progress ordering and
44
- * close the returned iterator on completion, cancellation or failure (for await handles this).
45
- */
46
- open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamItem>;
47
- }
20
+ resolveMetadata?: () => MongoReplicationStreamBatchMetadata;
21
+ }): MongoReplicationStreamBatch;
22
+ /**
23
+ * Filter complete data changes while retaining source batch boundaries and recovery bookkeeping.
24
+ * Return MongoReplicationFilterResult.KEEP to forward a change, EXCLUDE to omit it, or DELETE for an exit.
25
+ * Synthetic deletes retain the source identity, resume token and transaction metadata.
26
+ * The callback runs synchronously and lazily; failures and early exits close the underlying stream.
27
+ */
28
+ export declare function filterMongoReplicationStream({ batches, filter }: MongoReplicationFilterOptions): AsyncGenerator<MongoReplicationStreamBatch>;
48
29
  /**
49
30
  * Shared opening path for ongoing replication, snapshot barriers and resume validation.
50
31
  */
@@ -67,19 +48,27 @@ export declare function openMongoReplicationStream({ db, queryProvider, namespac
67
48
  skipInitialTimestamp?: boolean;
68
49
  options: RawChangeStreamOptions;
69
50
  onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
70
- }): AsyncIterableIterator<MongoReplicationStreamItem>;
51
+ }): AsyncIterableIterator<MongoReplicationStreamBatch>;
71
52
  /**
72
- * Parse envelopes once, retaining raw BSON row bodies and withholding progress during split reassembly.
53
+ * Creates a stateful parser for one raw MongoDB stream. Parses BSON entries, joins split-event
54
+ * fragments across batches, normalizes namespaces and skips events covered by legacy timestamps.
55
+ * After processing a batch, callers can request safe progress. Check completion on natural EOF,
56
+ * not cancellation.
73
57
  */
74
- export declare function readMongoReplicationStream({ batches, defaultSchema, multipleDatabases, startAfter, signal, logger, onBatch }: {
75
- batches: AsyncIterable<ChangeStreamBatch>;
76
- defaultSchema: string;
77
- multipleDatabases: boolean;
78
- startAfter?: mongo.Timestamp | null;
79
- signal?: AbortSignal;
80
- logger?: Logger;
58
+ export declare function createMongoReplicationStreamParser({ defaultSchema, multipleDatabases, startAfter, signal, logger }: MongoReplicationParserOptions): {
59
+ parse(rawEvent: Buffer): ProjectedChangeStreamDocument | null;
81
60
  /**
82
- * Transport accounting happens before parsing/filtering, including partial fragments and empty batches.
61
+ * Returns the given transport token if it is safe to persist, or undefined while a split event is unfinished.
83
62
  */
84
- onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
85
- }): AsyncGenerator<MongoReplicationStreamItem>;
63
+ getSafeResumeToken(resumeToken: mongo.ResumeToken): unknown;
64
+ assertComplete(): void;
65
+ };
66
+ /**
67
+ * Wrap raw batches in lazy event iterables. BSON parsing and split-event assembly happen
68
+ * as consumers request events, without building a parsed array. After exhaustion, each batch retains
69
+ * its own resume-token result even when later batches change the parser state.
70
+ * Consume each batch completely before requesting the next; returning early closes the source.
71
+ */
72
+ export declare function parseMongoReplicationStream({ batches, ...options }: MongoReplicationParserOptions & {
73
+ batches: AsyncIterable<RawMongoReplicationStreamBatch>;
74
+ }): AsyncGenerator<MongoReplicationStreamBatch>;
@@ -1,11 +1,106 @@
1
1
  import { ReplicationAssertionError } from '@powersync/lib-services-framework';
2
2
  import { buildMongoChangeStreamPipeline } from './MongoReplicationQueryProvider.js';
3
+ import { MongoReplicationFilterResult } from './MongoReplicationStreamTypes.js';
3
4
  import { parseChangeDocument, rawChangeStream } from './RawChangeStream.js';
5
+ import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
6
+ /**
7
+ * Wraps changes with a single-use iterator. Caches the token and metadata after successful exhaustion.
8
+ */
9
+ export function createMongoReplicationStreamBatch({ events, resolveResumeToken, resolveMetadata = () => ({ filteredCount: null }) }) {
10
+ let started = false;
11
+ let complete = false;
12
+ let resumeToken;
13
+ let metadata;
14
+ function assertComplete() {
15
+ // Progress is only safe to read after every event has been consumed and finalization succeeds.
16
+ if (!complete) {
17
+ throw new ReplicationAssertionError('Consume all batch events before requesting token or metadata');
18
+ }
19
+ }
20
+ return {
21
+ events: {
22
+ [Symbol.iterator]() {
23
+ // A second pass would repeat parsing/filtering against state that has already advanced.
24
+ if (started) {
25
+ throw new ReplicationAssertionError('Batch events can only be consumed once');
26
+ }
27
+ started = true;
28
+ return iterate();
29
+ }
30
+ },
31
+ getResumeToken() {
32
+ assertComplete();
33
+ return resumeToken;
34
+ },
35
+ get metadata() {
36
+ assertComplete();
37
+ return metadata;
38
+ }
39
+ };
40
+ function* iterate() {
41
+ // Delegate lazily; an error or early return skips the finalization below.
42
+ yield* events;
43
+ // Cache this batch's final results before later batches can change the parser or filter state.
44
+ resumeToken = resolveResumeToken();
45
+ metadata = Object.freeze({ ...resolveMetadata() });
46
+ complete = true;
47
+ }
48
+ }
49
+ /**
50
+ * Filter complete data changes while retaining source batch boundaries and recovery bookkeeping.
51
+ * Return MongoReplicationFilterResult.KEEP to forward a change, EXCLUDE to omit it, or DELETE for an exit.
52
+ * Synthetic deletes retain the source identity, resume token and transaction metadata.
53
+ * The callback runs synchronously and lazily; failures and early exits close the underlying stream.
54
+ */
55
+ export async function* filterMongoReplicationStream({ batches, filter }) {
56
+ for await (const sourceBatch of batches) {
57
+ let filteredCount = 0;
58
+ const batch = createMongoReplicationStreamBatch({
59
+ events: filterEvents(),
60
+ resolveResumeToken: () => sourceBatch.getResumeToken(),
61
+ resolveMetadata: () => ({ filteredCount: (sourceBatch.metadata.filteredCount ?? 0) + filteredCount })
62
+ });
63
+ yield batch;
64
+ // Assert that the consumer exhausted this batch before requesting the next one, even if every event
65
+ // was filtered out. The token value is unused; getResumeToken() throws if consumption is incomplete.
66
+ batch.getResumeToken();
67
+ function* filterEvents() {
68
+ for (const change of sourceBatch.events) {
69
+ const { event } = change;
70
+ if ((event.operationType != 'insert' &&
71
+ event.operationType != 'update' &&
72
+ event.operationType != 'replace' &&
73
+ event.operationType != 'delete') ||
74
+ event.ns.coll == CHECKPOINTS_COLLECTION) {
75
+ yield change;
76
+ continue;
77
+ }
78
+ const result = filter(event);
79
+ if (result == MongoReplicationFilterResult.EXCLUDE) {
80
+ filteredCount++;
81
+ }
82
+ else if (result == MongoReplicationFilterResult.DELETE) {
83
+ yield { ...change, event: { ...event, operationType: 'delete' } };
84
+ }
85
+ else {
86
+ yield change;
87
+ }
88
+ }
89
+ }
90
+ }
91
+ }
4
92
  /**
5
93
  * Shared opening path for ongoing replication, snapshot barriers and resume validation.
6
94
  */
7
95
  export function openMongoReplicationStream({ db, queryProvider, namespaceFilter, isDocumentDb, usePostImages, position, skipInitialTimestamp = false, options, onBatch }) {
8
- const open = ({ pipelineStages = [], imageOptions = {} }) => {
96
+ const parsing = {
97
+ defaultSchema: db.databaseName,
98
+ multipleDatabases: namespaceFilter.multipleDatabases,
99
+ startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
100
+ signal: options.signal,
101
+ logger: options.logger
102
+ };
103
+ const openRaw = async function* ({ pipelineStages = [], imageOptions = {} }) {
9
104
  const clusterScope = isDocumentDb || namespaceFilter.multipleDatabases;
10
105
  const streamOptions = {
11
106
  ...imageOptions,
@@ -27,31 +122,27 @@ export function openMongoReplicationStream({ db, queryProvider, namespaceFilter,
27
122
  pipelineStages,
28
123
  includeSplitLargeEvent: !isDocumentDb
29
124
  });
30
- return readMongoReplicationStream({
31
- batches: rawChangeStream(clusterScope ? db.client.db('admin') : db, pipeline, options),
32
- defaultSchema: db.databaseName,
33
- multipleDatabases: namespaceFilter.multipleDatabases,
34
- // An exact token can fall inside a transaction. Later events may share its timestamp and must
35
- // still be delivered. Timestamp deduplication applies only to legacy positions without tokens.
36
- startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
37
- signal: options.signal,
38
- logger: options.logger,
39
- onBatch
40
- });
125
+ for await (const batch of rawChangeStream(clusterScope ? db.client.db('admin') : db, pipeline, options)) {
126
+ options.signal?.throwIfAborted();
127
+ using batchScope = onBatch?.(batch) || undefined;
128
+ yield { events: batch.events, resumeToken: batch.resumeToken };
129
+ }
41
130
  };
42
- return queryProvider.openChangeStream({ open });
131
+ return queryProvider.openChangeStream({
132
+ open: (options) => parseMongoReplicationStream({ batches: openRaw(options), ...parsing })
133
+ });
43
134
  }
44
135
  /**
45
- * Parse envelopes once, retaining raw BSON row bodies and withholding progress during split reassembly.
136
+ * Creates a stateful parser for one raw MongoDB stream. Parses BSON entries, joins split-event
137
+ * fragments across batches, normalizes namespaces and skips events covered by legacy timestamps.
138
+ * After processing a batch, callers can request safe progress. Check completion on natural EOF,
139
+ * not cancellation.
46
140
  */
47
- export async function* readMongoReplicationStream({ batches, defaultSchema, multipleDatabases, startAfter, signal, logger, onBatch }) {
141
+ export function createMongoReplicationStreamParser({ defaultSchema, multipleDatabases, startAfter, signal, logger }) {
48
142
  let splitDocument = null;
49
143
  let flexDbNameWorkaroundLogged = false;
50
- for await (const batch of batches) {
51
- signal?.throwIfAborted();
52
- // The optional scope measures batch processing, excluding the next network wait.
53
- using batchScope = onBatch?.(batch) || undefined;
54
- for (const [index, rawEvent] of batch.events.entries()) {
144
+ return {
145
+ parse(rawEvent) {
55
146
  signal?.throwIfAborted();
56
147
  let event = parseChangeDocument(rawEvent);
57
148
  const fragment = event.splitEvent;
@@ -65,16 +156,18 @@ export async function* readMongoReplicationStream({ batches, defaultSchema, mult
65
156
  throw new ReplicationAssertionError(`Unexpected splitEvent: ${JSON.stringify(fragment)}`);
66
157
  }
67
158
  splitDocument = splitDocument == null ? event : Object.assign(splitDocument, event);
68
- if (fragment.fragment != fragment.of)
69
- continue;
159
+ if (fragment.fragment != fragment.of) {
160
+ return null;
161
+ }
70
162
  event = splitDocument;
71
163
  splitDocument = null;
72
164
  }
73
165
  else if (splitDocument != null) {
74
166
  throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
75
167
  }
76
- if (startAfter != null && event.clusterTime?.lte(startAfter))
77
- continue;
168
+ if (startAfter != null && event.clusterTime?.lte(startAfter)) {
169
+ return null;
170
+ }
78
171
  if (!multipleDatabases &&
79
172
  'ns' in event &&
80
173
  event.ns.db != defaultSchema &&
@@ -88,14 +181,48 @@ export async function* readMongoReplicationStream({ batches, defaultSchema, mult
88
181
  logger?.warn(`Incorrect DB name in change stream: ${originalDatabase}. Changed to ${defaultSchema}.`);
89
182
  }
90
183
  }
91
- yield { type: 'change', event, hasBufferedChanges: index < batch.events.length - 1 };
92
- }
93
- if (splitDocument == null) {
94
- yield { type: 'progress', resumeToken: batch.resumeToken, filteredCount: 0 };
184
+ return event;
185
+ },
186
+ /**
187
+ * Returns the given transport token if it is safe to persist, or undefined while a split event is unfinished.
188
+ */
189
+ getSafeResumeToken(resumeToken) {
190
+ return splitDocument == null ? resumeToken : undefined;
191
+ },
192
+ assertComplete() {
193
+ if (splitDocument != null) {
194
+ throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
195
+ }
95
196
  }
197
+ };
198
+ }
199
+ /**
200
+ * Wrap raw batches in lazy event iterables. BSON parsing and split-event assembly happen
201
+ * as consumers request events, without building a parsed array. After exhaustion, each batch retains
202
+ * its own resume-token result even when later batches change the parser state.
203
+ * Consume each batch completely before requesting the next; returning early closes the source.
204
+ */
205
+ export async function* parseMongoReplicationStream({ batches, ...options }) {
206
+ const parser = createMongoReplicationStreamParser(options);
207
+ for await (const sourceBatch of batches) {
208
+ options.signal?.throwIfAborted();
209
+ const batch = createMongoReplicationStreamBatch({
210
+ events: parseEvents(sourceBatch.events),
211
+ resolveResumeToken: () => parser.getSafeResumeToken(sourceBatch.resumeToken)
212
+ });
213
+ yield batch;
214
+ // Split-event state is shared across batches, so the consumer must finish this one before the next is read.
215
+ // This also covers empty batches.
216
+ batch.getResumeToken();
96
217
  }
97
- if (splitDocument != null) {
98
- throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
218
+ parser.assertComplete();
219
+ function* parseEvents(rawEvents) {
220
+ for (let index = 0; index < rawEvents.length; index++) {
221
+ const event = parser.parse(rawEvents[index]);
222
+ if (event != null) {
223
+ yield { event, hasBufferedChanges: index < rawEvents.length - 1 };
224
+ }
225
+ }
99
226
  }
100
227
  }
101
228
  //# sourceMappingURL=MongoReplicationStream.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"MongoReplicationStream.js","sourceRoot":"","sources":["../../src/replication/MongoReplicationStream.ts"],"names":[],"mappings":"AACA,OAAO,EAAU,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAE,8BAA8B,EAAiC,MAAM,oCAAoC,CAAC;AACnH,OAAO,EAEL,mBAAmB,EAEnB,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAkD9B;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,EACzC,EAAE,EACF,aAAa,EACb,eAAe,EACf,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,oBAAoB,GAAG,KAAK,EAC5B,OAAO,EACP,OAAO,EAcR;IACC,MAAM,IAAI,GAA0C,CAAC,EAAE,cAAc,GAAG,EAAE,EAAE,YAAY,GAAG,EAAE,EAAE,EAAE,EAAE;QACjG,MAAM,YAAY,GAAG,YAAY,IAAI,eAAe,CAAC,iBAAiB,CAAC;QACvE,MAAM,aAAa,GAAmB;YACpC,GAAG,YAAY;YACf,YAAY,EAAE,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc;YAC1E,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC;QACF,mGAAmG;QACnG,4DAA4D;QAC5D,IAAI,QAAQ,EAAE,WAAW,IAAI,IAAI,EAAE,CAAC;YAClC,aAAa,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;QACnD,CAAC;aAAM,IAAI,QAAQ,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC;YACxC,aAAa,CAAC,oBAAoB,GAAG,QAAQ,CAAC,UAAU,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAG,8BAA8B,CAAC;YAC9C,aAAa;YACb,cAAc,EAAE,eAAe,CAAC,MAAM;YACtC,cAAc;YACd,sBAAsB,EAAE,CAAC,YAAY;SACtC,CAAC,CAAC;QACH,OAAO,0BAA0B,CAAC;YAChC,OAAO,EAAE,eAAe,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC;YACtF,aAAa,EAAE,EAAE,CAAC,YAAY;YAC9B,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;YACpD,8FAA8F;YAC9F,+FAA+F;YAC/F,UAAU,EAAE,oBAAoB,IAAI,QAAQ,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS;YACpG,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO;SACR,CAAC,CAAC;IACL,CAAC,CAAC;IACF,OAAO,aAAa,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,0BAA0B,CAAC,EAChD,OAAO,EACP,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,MAAM,EACN,MAAM,EACN,OAAO,EAYR;IACC,IAAI,aAAa,GAAyC,IAAI,CAAC;IAC/D,IAAI,0BAA0B,GAAG,KAAK,CAAC;IACvC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAClC,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,iFAAiF;QACjF,MAAM,UAAU,GAAG,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC;QACjD,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,IAAI,KAAK,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC;YAClC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,gGAAgG;gBAChG,4FAA4F;gBAC5F,MAAM,gBAAgB,GAAG,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,UAAW,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC5F,IACE,QAAQ,CAAC,QAAQ,IAAI,gBAAgB;oBACrC,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC,EAAE;oBAC/B,CAAC,aAAa,IAAI,IAAI,IAAI,QAAQ,CAAC,EAAE,IAAI,aAAa,CAAC,UAAW,CAAC,EAAE,CAAC,EACtE,CAAC;oBACD,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAC5F,CAAC;gBACD,aAAa,GAAG,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;gBACpF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE;oBAAE,SAAS;gBAC/C,KAAK,GAAG,aAAa,CAAC;gBACtB,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;gBACjC,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC5G,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,UAAU,CAAC;gBAAE,SAAS;YAEvE,IACE,CAAC,iBAAiB;gBAClB,IAAI,IAAI,KAAK;gBACb,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,aAAa;gBAC5B,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC,EACzC,CAAC;gBACD,2FAA2F;gBAC3F,4FAA4F;gBAC5F,MAAM,gBAAgB,GAAG,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;gBACrC,KAAK,CAAC,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC;gBAC5B,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBAChC,0BAA0B,GAAG,IAAI,CAAC;oBAClC,MAAM,EAAE,IAAI,CAAC,uCAAuC,gBAAgB,gBAAgB,aAAa,GAAG,CAAC,CAAC;gBACxG,CAAC;YACH,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvF,CAAC;QACD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;QAC/E,CAAC;IACH,CAAC;IACD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"MongoReplicationStream.js","sourceRoot":"","sources":["../../src/replication/MongoReplicationStream.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,8BAA8B,EAAiC,MAAM,oCAAoC,CAAC;AACnH,OAAO,EAEL,4BAA4B,EAO7B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAEL,mBAAmB,EAEnB,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE;;GAEG;AACH,MAAM,UAAU,iCAAiC,CAAC,EAChD,MAAM,EACN,kBAAkB,EAClB,eAAe,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAclD;IACC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,WAA0C,CAAC;IAC/C,IAAI,QAA6C,CAAC;IAClD,SAAS,cAAc;QACrB,+FAA+F;QAC/F,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,yBAAyB,CAAC,8DAA8D,CAAC,CAAC;QACtG,CAAC;IACH,CAAC;IACD,OAAO;QACL,MAAM,EAAE;YACN,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACf,wFAAwF;gBACxF,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;gBAChF,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,OAAO,EAAE,CAAC;YACnB,CAAC;SACF;QACD,cAAc;YACZ,cAAc,EAAE,CAAC;YACjB,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,QAAQ;YACV,cAAc,EAAE,CAAC;YACjB,OAAO,QAAQ,CAAC;QAClB,CAAC;KACF,CAAC;IAEF,QAAQ,CAAC,CAAC,OAAO;QACf,0EAA0E;QAC1E,KAAK,CAAC,CAAC,MAAM,CAAC;QACd,+FAA+F;QAC/F,WAAW,GAAG,kBAAkB,EAAE,CAAC;QACnC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,eAAe,EAAE,EAAE,CAAC,CAAC;QACnD,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,4BAA4B,CAAC,EAClD,OAAO,EACP,MAAM,EACwB;IAC9B,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,OAAO,EAAE,CAAC;QACxC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,KAAK,GAAG,iCAAiC,CAAC;YAC9C,MAAM,EAAE,YAAY,EAAE;YACtB,kBAAkB,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,cAAc,EAAE;YACtD,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,aAAa,EAAE,CAAC;SACtG,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;QACZ,oGAAoG;QACpG,qGAAqG;QACrG,KAAK,CAAC,cAAc,EAAE,CAAC;QAEvB,QAAQ,CAAC,CAAC,YAAY;YACpB,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;gBACzB,IACE,CAAC,KAAK,CAAC,aAAa,IAAI,QAAQ;oBAC9B,KAAK,CAAC,aAAa,IAAI,QAAQ;oBAC/B,KAAK,CAAC,aAAa,IAAI,SAAS;oBAChC,KAAK,CAAC,aAAa,IAAI,QAAQ,CAAC;oBAClC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,sBAAsB,EACvC,CAAC;oBACD,MAAM,MAAM,CAAC;oBACb,SAAS;gBACX,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC7B,IAAI,MAAM,IAAI,4BAA4B,CAAC,OAAO,EAAE,CAAC;oBACnD,aAAa,EAAE,CAAC;gBAClB,CAAC;qBAAM,IAAI,MAAM,IAAI,4BAA4B,CAAC,MAAM,EAAE,CAAC;oBACzD,MAAM,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,CAAC;gBACpE,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC;gBACf,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,EACzC,EAAE,EACF,aAAa,EACb,eAAe,EACf,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,oBAAoB,GAAG,KAAK,EAC5B,OAAO,EACP,OAAO,EAcR;IACC,MAAM,OAAO,GAAG;QACd,aAAa,EAAE,EAAE,CAAC,YAAY;QAC9B,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;QACpD,UAAU,EAAE,oBAAoB,IAAI,QAAQ,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS;QACpG,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,SAAS,CAAC,EAAE,EAC/B,cAAc,GAAG,EAAE,EACnB,YAAY,GAAG,EAAE,EACa;QAC9B,MAAM,YAAY,GAAG,YAAY,IAAI,eAAe,CAAC,iBAAiB,CAAC;QACvE,MAAM,aAAa,GAAmB;YACpC,GAAG,YAAY;YACf,YAAY,EAAE,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc;YAC1E,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC;QACF,mGAAmG;QACnG,4DAA4D;QAC5D,IAAI,QAAQ,EAAE,WAAW,IAAI,IAAI,EAAE,CAAC;YAClC,aAAa,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;QACnD,CAAC;aAAM,IAAI,QAAQ,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC;YACxC,aAAa,CAAC,oBAAoB,GAAG,QAAQ,CAAC,UAAU,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAG,8BAA8B,CAAC;YAC9C,aAAa;YACb,cAAc,EAAE,eAAe,CAAC,MAAM;YACtC,cAAc;YACd,sBAAsB,EAAE,CAAC,YAAY;SACtC,CAAC,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,eAAe,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;YACxG,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC;YACjD,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QACjE,CAAC;IACH,CAAC,CAAC;IAEF,OAAO,aAAa,CAAC,gBAAgB,CAAC;QACpC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC;KAC1F,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kCAAkC,CAAC,EACjD,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,MAAM,EACN,MAAM,EACwB;IAC9B,IAAI,aAAa,GAAyC,IAAI,CAAC;IAC/D,IAAI,0BAA0B,GAAG,KAAK,CAAC;IACvC,OAAO;QACL,KAAK,CAAC,QAAgB;YACpB,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,IAAI,KAAK,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC;YAClC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,gGAAgG;gBAChG,4FAA4F;gBAC5F,MAAM,gBAAgB,GAAG,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,UAAW,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC5F,IACE,QAAQ,CAAC,QAAQ,IAAI,gBAAgB;oBACrC,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC,EAAE;oBAC/B,CAAC,aAAa,IAAI,IAAI,IAAI,QAAQ,CAAC,EAAE,IAAI,aAAa,CAAC,UAAW,CAAC,EAAE,CAAC,EACtE,CAAC;oBACD,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAC5F,CAAC;gBACD,aAAa,GAAG,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;gBACpF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACrC,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,KAAK,GAAG,aAAa,CAAC;gBACtB,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;gBACjC,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC5G,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7D,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IACE,CAAC,iBAAiB;gBAClB,IAAI,IAAI,KAAK;gBACb,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,aAAa;gBAC5B,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC,EACzC,CAAC;gBACD,2FAA2F;gBAC3F,4FAA4F;gBAC5F,MAAM,gBAAgB,GAAG,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;gBACrC,KAAK,CAAC,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC;gBAC5B,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBAChC,0BAA0B,GAAG,IAAI,CAAC;oBAClC,MAAM,EAAE,IAAI,CAAC,uCAAuC,gBAAgB,gBAAgB,aAAa,GAAG,CAAC,CAAC;gBACxG,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QACD;;WAEG;QACH,kBAAkB,CAAC,WAA8B;YAC/C,OAAO,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,CAAC;QACD,cAAc;YACZ,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC5G,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,2BAA2B,CAAC,EACjD,OAAO,EACP,GAAG,OAAO,EAGX;IACC,MAAM,MAAM,GAAG,kCAAkC,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,OAAO,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,iCAAiC,CAAC;YAC9C,MAAM,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC;YACvC,kBAAkB,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,WAAW,CAAC,WAAW,CAAC;SAC7E,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;QACZ,4GAA4G;QAC5G,kCAAkC;QAClC,KAAK,CAAC,cAAc,EAAE,CAAC;IACzB,CAAC;IACD,MAAM,CAAC,cAAc,EAAE,CAAC;IAExB,QAAQ,CAAC,CAAC,WAAW,CAAC,SAAmB;QACvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpE,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -0,0 +1,157 @@
1
+ import type { mongo } from '@powersync/lib-service-mongodb';
2
+ import type { Logger } from '@powersync/lib-services-framework';
3
+ import type { ProjectedChangeStreamDocument } from './RawChangeStream.js';
4
+ /**
5
+ * An unparsed MongoDB response batch. Consumers parse its entries incrementally and retain unfinished
6
+ * split events across batches before deciding whether its resume token is safe to persist.
7
+ */
8
+ export interface RawMongoReplicationStreamBatch {
9
+ /**
10
+ * Original BSON entries in source order, including possible split-event fragments.
11
+ */
12
+ events: Buffer[];
13
+ /**
14
+ * Transport position after this batch. It may cover an incomplete event and is not yet safe progress.
15
+ */
16
+ resumeToken: mongo.ResumeToken;
17
+ }
18
+ /**
19
+ * A complete event and its position within the original source batch.
20
+ */
21
+ export interface MongoReplicationStreamChange {
22
+ event: ProjectedChangeStreamDocument;
23
+ /**
24
+ * MongoDB returned more entries after this event, even if a provider later excludes them or the
25
+ * parser retains them as unfinished split fragments. For [checkpoint, excluded change], preserve
26
+ * true on the checkpoint so ChangeStream groups commits instead of treating it as caught up.
27
+ * Providers must preserve this value when forwarding or transforming events.
28
+ */
29
+ hasBufferedChanges: boolean;
30
+ }
31
+ /**
32
+ * Statistics for one fully consumed batch, independent of whether its resume token is safe.
33
+ */
34
+ export interface MongoReplicationStreamBatchMetadata {
35
+ /**
36
+ * Complete events excluded from this batch. Null means no filtering was applied; zero means
37
+ * filtering was applied but no events were excluded. Counts do not carry across batches.
38
+ */
39
+ readonly filteredCount: number | null;
40
+ }
41
+ /**
42
+ * A single-use batch of complete events, parsed or filtered lazily as the consumer iterates.
43
+ * Exhaust events before reading its resume token, metadata or the next batch.
44
+ */
45
+ export interface MongoReplicationStreamBatch {
46
+ /**
47
+ * Complete changes in source order, parsed or filtered lazily during synchronous iteration.
48
+ * Consume once and fully. It may be empty after filtering or while assembling a split event.
49
+ */
50
+ events: Iterable<MongoReplicationStreamChange>;
51
+ /**
52
+ * Returns the safe resume token after events are fully consumed, or undefined while a split event
53
+ * is unfinished. Throws before exhaustion. The result remains stable after later batches.
54
+ */
55
+ getResumeToken(): mongo.ResumeToken | undefined;
56
+ /**
57
+ * Final statistics for this batch, available even without a safe resume token.
58
+ * Throws before events are fully consumed; the result is immutable and cached afterward.
59
+ */
60
+ readonly metadata: MongoReplicationStreamBatchMetadata;
61
+ }
62
+ /**
63
+ * Provider-controlled additions to the shared cursor setup. Source scope, resume position, post-image
64
+ * requirements and cancellation remain controlled by the MongoDB replication module.
65
+ */
66
+ export interface MongoReplicationStreamOptions {
67
+ /**
68
+ * Inserted after namespace selection and before the final large-event split stage.
69
+ */
70
+ pipelineStages?: mongo.Document[];
71
+ /**
72
+ * Request pre-images without overriding the reader's post-image, resume or scope options.
73
+ */
74
+ imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
75
+ }
76
+ /**
77
+ * Bound source opener supplied to a query provider for one stream invocation.
78
+ * Providers may wrap the lazy event iterables to filter or transform complete changes.
79
+ */
80
+ export interface MongoReplicationStreamContext {
81
+ /**
82
+ * Open batches whose events are parsed lazily, with cursor scope, position, cancellation and
83
+ * accounting already bound. Each call owns independent split-event assembly and parser state.
84
+ * Events are complete; progress is withheld while a split event is unfinished.
85
+ * Close the iterator on completion, cancellation or failure (for await handles this).
86
+ */
87
+ open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamBatch>;
88
+ }
89
+ /**
90
+ * Source context for lazy parsing of raw BSON events in default and filtering providers.
91
+ * These options control namespace normalization, legacy timestamp skipping and cancellation; split-event
92
+ * state belongs to each parser instance and must not be shared between independently opened streams.
93
+ */
94
+ export interface MongoReplicationParserOptions {
95
+ /**
96
+ * Expected database name when correcting Atlas Flex's prefixed namespaces in single-database mode.
97
+ */
98
+ defaultSchema: string;
99
+ /**
100
+ * Preserve database names when the stream covers multiple databases instead of normalizing them.
101
+ */
102
+ multipleDatabases: boolean;
103
+ /**
104
+ * Exclude complete events at or before this legacy resume timestamp. Leave unset for exact-token
105
+ * resumes, where later events can share the same timestamp, and for snapshot barrier probes.
106
+ */
107
+ startAfter?: mongo.Timestamp | null;
108
+ /**
109
+ * Abort parsing when the owning replication attempt is cancelled. Resolving a batch's progress is not
110
+ * interrupted, so consumers must check the signal themselves before persisting a token.
111
+ */
112
+ signal?: AbortSignal;
113
+ /**
114
+ * Receives the namespace-correction warning, at most once per parser instance.
115
+ */
116
+ logger?: Logger;
117
+ }
118
+ /**
119
+ * A complete data change eligible for filtering. Checkpoints and DDL bypass the filter.
120
+ */
121
+ export type MongoReplicationDataChange = Extract<ProjectedChangeStreamDocument, {
122
+ operationType: 'insert' | 'update' | 'replace' | 'delete';
123
+ }>;
124
+ /**
125
+ * The action to apply to a complete data change after evaluating filter membership.
126
+ */
127
+ export declare enum MongoReplicationFilterResult {
128
+ /**
129
+ * Forward the original change.
130
+ */
131
+ KEEP = "keep",
132
+ /**
133
+ * Omit the change while retaining its batch's recovery progress.
134
+ */
135
+ EXCLUDE = "exclude",
136
+ /**
137
+ * Emit a delete because the document left the filtered set.
138
+ */
139
+ DELETE = "delete"
140
+ }
141
+ /**
142
+ * Synchronously decide how to replicate a complete data change. Checkpoints and DDL bypass this callback.
143
+ */
144
+ export type MongoReplicationFilter = (event: MongoReplicationDataChange) => MongoReplicationFilterResult;
145
+ /**
146
+ * Source batches and membership decisions for the shared replication filter.
147
+ */
148
+ export interface MongoReplicationFilterOptions {
149
+ /**
150
+ * Ordered source batches whose boundaries and recovery progress must be preserved.
151
+ */
152
+ batches: AsyncIterable<MongoReplicationStreamBatch>;
153
+ /**
154
+ * Decide whether to keep, exclude or delete each complete data change.
155
+ */
156
+ filter: MongoReplicationFilter;
157
+ }
@@ -0,0 +1,19 @@
1
+ /**
2
+ * The action to apply to a complete data change after evaluating filter membership.
3
+ */
4
+ export var MongoReplicationFilterResult;
5
+ (function (MongoReplicationFilterResult) {
6
+ /**
7
+ * Forward the original change.
8
+ */
9
+ MongoReplicationFilterResult["KEEP"] = "keep";
10
+ /**
11
+ * Omit the change while retaining its batch's recovery progress.
12
+ */
13
+ MongoReplicationFilterResult["EXCLUDE"] = "exclude";
14
+ /**
15
+ * Emit a delete because the document left the filtered set.
16
+ */
17
+ MongoReplicationFilterResult["DELETE"] = "delete";
18
+ })(MongoReplicationFilterResult || (MongoReplicationFilterResult = {}));
19
+ //# sourceMappingURL=MongoReplicationStreamTypes.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"MongoReplicationStreamTypes.js","sourceRoot":"","sources":["../../src/replication/MongoReplicationStreamTypes.ts"],"names":[],"mappings":"AAqIA;;GAEG;AACH,MAAM,CAAN,IAAY,4BAaX;AAbD,WAAY,4BAA4B;IACtC;;OAEG;IACH,6CAAa,CAAA;IACb;;OAEG;IACH,mDAAmB,CAAA;IACnB;;OAEG;IACH,iDAAiB,CAAA;AACnB,CAAC,EAbW,4BAA4B,KAA5B,4BAA4B,QAavC"}