@powersync/service-module-mongodb 0.0.0-dev-20261007110800 → 0.0.0-dev-20261015132441
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -11
- package/benchmarks/env.ts +11 -0
- package/benchmarks/replication_branch_throughput.benchmark.ts +240 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +5 -0
- package/dist/api/MongoRouteAPIAdapter.js +11 -4
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/module/MongoModule.js +1 -0
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +2 -1
- package/dist/replication/ChangeStream.js +164 -150
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +17 -9
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/MongoReplicationQueryProvider.d.ts +9 -11
- package/dist/replication/MongoReplicationQueryProvider.js.map +1 -1
- package/dist/replication/MongoReplicationStream.d.ts +37 -48
- package/dist/replication/MongoReplicationStream.js +157 -30
- package/dist/replication/MongoReplicationStream.js.map +1 -1
- package/dist/replication/MongoReplicationStreamTypes.d.ts +157 -0
- package/dist/replication/MongoReplicationStreamTypes.js +19 -0
- package/dist/replication/MongoReplicationStreamTypes.js.map +1 -0
- package/dist/replication/MongoSnapshotter.js +59 -30
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/estimateFilteredSnapshotCount.d.ts +12 -0
- package/dist/replication/estimateFilteredSnapshotCount.js +79 -0
- package/dist/replication/estimateFilteredSnapshotCount.js.map +1 -0
- package/dist/replication/replication-index.d.ts +1 -0
- package/dist/replication/replication-index.js +1 -0
- package/dist/replication/replication-index.js.map +1 -1
- package/package.json +11 -10
- package/src/api/MongoRouteAPIAdapter.ts +12 -5
- package/src/module/MongoModule.ts +1 -0
- package/src/replication/ChangeStream.ts +183 -173
- package/src/replication/ChangeStreamReplicationJob.ts +16 -9
- package/src/replication/MongoReplicationQueryProvider.ts +9 -11
- package/src/replication/MongoReplicationStream.ts +196 -83
- package/src/replication/MongoReplicationStreamTypes.ts +169 -0
- package/src/replication/MongoSnapshotter.ts +64 -33
- package/src/replication/estimateFilteredSnapshotCount.ts +93 -0
- package/src/replication/replication-index.ts +1 -0
- package/test/src/change_event_memory.test.ts +33 -0
- package/test/src/change_stream_abort.test.ts +110 -0
- package/test/src/filtering_example.test.ts +65 -0
- package/test/src/filtering_lifecycle.test.ts +101 -0
- package/test/src/query_provider_availability.test.ts +1 -0
- package/test/src/query_provider_diagnostics.test.ts +55 -0
- package/test/src/replication_stream.test.ts +411 -34
- package/test/src/snapshot_count.test.ts +146 -0
- package/test/src/snapshot_count_integration.test.ts +70 -0
- package/test/src/snapshot_query_provider.test.ts +21 -2
- package/test/src/stream_progress.test.ts +62 -44
- package/test/tsconfig.json +2 -2
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.benchmark.config.ts +15 -0
|
@@ -1,50 +1,31 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { Logger } from '@powersync/lib-services-framework';
|
|
3
2
|
import { MongoReplicationQueryProvider } from './MongoReplicationQueryProvider.js';
|
|
3
|
+
import { MongoReplicationFilterOptions, MongoReplicationParserOptions, MongoReplicationStreamBatch, MongoReplicationStreamBatchMetadata, MongoReplicationStreamChange, RawMongoReplicationStreamBatch } from './MongoReplicationStreamTypes.js';
|
|
4
4
|
import { ChangeStreamBatch, ProjectedChangeStreamDocument, RawChangeStreamOptions } from './RawChangeStream.js';
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* Wraps changes with a single-use iterator. Caches the token and metadata after successful exhaustion.
|
|
7
7
|
*/
|
|
8
|
-
export
|
|
9
|
-
type: 'change';
|
|
10
|
-
event: ProjectedChangeStreamDocument;
|
|
8
|
+
export declare function createMongoReplicationStreamBatch({ events, resolveResumeToken, resolveMetadata }: {
|
|
11
9
|
/**
|
|
12
|
-
*
|
|
10
|
+
* Complete changes, optionally produced by a lazy parsing or filtering generator.
|
|
13
11
|
*/
|
|
14
|
-
|
|
15
|
-
} | {
|
|
16
|
-
type: 'progress';
|
|
12
|
+
events: Iterable<MongoReplicationStreamChange>;
|
|
17
13
|
/**
|
|
18
|
-
*
|
|
14
|
+
* Compute a safe resume token once after successful exhaustion, using the final parser state.
|
|
19
15
|
*/
|
|
20
|
-
|
|
16
|
+
resolveResumeToken: () => mongo.ResumeToken | undefined;
|
|
21
17
|
/**
|
|
22
|
-
*
|
|
18
|
+
* Compute this batch's final statistics after exhaustion. Defaults to no filtering.
|
|
23
19
|
*/
|
|
24
|
-
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
*/
|
|
34
|
-
imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
|
|
35
|
-
}
|
|
36
|
-
export interface MongoReplicationStreamContext {
|
|
37
|
-
/**
|
|
38
|
-
* Open the source reader with the current position, namespace, deadlines and cancellation already bound.
|
|
39
|
-
* The reader returns parsed, reassembled changes followed by safe progress boundaries. A provider can
|
|
40
|
-
* add stages here and wrap the result to discard marked events or turn filter exits into deletes.
|
|
41
|
-
* Supplying this callback keeps cursor setup shared with snapshot barriers and resume validation.
|
|
42
|
-
*
|
|
43
|
-
* The default provider simply returns open({}). Wrappers must preserve event/progress ordering and
|
|
44
|
-
* close the returned iterator on completion, cancellation or failure (for await handles this).
|
|
45
|
-
*/
|
|
46
|
-
open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamItem>;
|
|
47
|
-
}
|
|
20
|
+
resolveMetadata?: () => MongoReplicationStreamBatchMetadata;
|
|
21
|
+
}): MongoReplicationStreamBatch;
|
|
22
|
+
/**
|
|
23
|
+
* Filter complete data changes while retaining source batch boundaries and recovery bookkeeping.
|
|
24
|
+
* Return MongoReplicationFilterResult.KEEP to forward a change, EXCLUDE to omit it, or DELETE for an exit.
|
|
25
|
+
* Synthetic deletes retain the source identity, resume token and transaction metadata.
|
|
26
|
+
* The callback runs synchronously and lazily; failures and early exits close the underlying stream.
|
|
27
|
+
*/
|
|
28
|
+
export declare function filterMongoReplicationStream({ batches, filter }: MongoReplicationFilterOptions): AsyncGenerator<MongoReplicationStreamBatch>;
|
|
48
29
|
/**
|
|
49
30
|
* Shared opening path for ongoing replication, snapshot barriers and resume validation.
|
|
50
31
|
*/
|
|
@@ -67,19 +48,27 @@ export declare function openMongoReplicationStream({ db, queryProvider, namespac
|
|
|
67
48
|
skipInitialTimestamp?: boolean;
|
|
68
49
|
options: RawChangeStreamOptions;
|
|
69
50
|
onBatch?: (batch: ChangeStreamBatch) => Disposable | void;
|
|
70
|
-
}): AsyncIterableIterator<
|
|
51
|
+
}): AsyncIterableIterator<MongoReplicationStreamBatch>;
|
|
71
52
|
/**
|
|
72
|
-
*
|
|
53
|
+
* Creates a stateful parser for one raw MongoDB stream. Parses BSON entries, joins split-event
|
|
54
|
+
* fragments across batches, normalizes namespaces and skips events covered by legacy timestamps.
|
|
55
|
+
* After processing a batch, callers can request safe progress. Check completion on natural EOF,
|
|
56
|
+
* not cancellation.
|
|
73
57
|
*/
|
|
74
|
-
export declare function
|
|
75
|
-
|
|
76
|
-
defaultSchema: string;
|
|
77
|
-
multipleDatabases: boolean;
|
|
78
|
-
startAfter?: mongo.Timestamp | null;
|
|
79
|
-
signal?: AbortSignal;
|
|
80
|
-
logger?: Logger;
|
|
58
|
+
export declare function createMongoReplicationStreamParser({ defaultSchema, multipleDatabases, startAfter, signal, logger }: MongoReplicationParserOptions): {
|
|
59
|
+
parse(rawEvent: Buffer): ProjectedChangeStreamDocument | null;
|
|
81
60
|
/**
|
|
82
|
-
*
|
|
61
|
+
* Returns the given transport token if it is safe to persist, or undefined while a split event is unfinished.
|
|
83
62
|
*/
|
|
84
|
-
|
|
85
|
-
|
|
63
|
+
getSafeResumeToken(resumeToken: mongo.ResumeToken): unknown;
|
|
64
|
+
assertComplete(): void;
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* Wrap raw batches in lazy event iterables. BSON parsing and split-event assembly happen
|
|
68
|
+
* as consumers request events, without building a parsed array. After exhaustion, each batch retains
|
|
69
|
+
* its own resume-token result even when later batches change the parser state.
|
|
70
|
+
* Consume each batch completely before requesting the next; returning early closes the source.
|
|
71
|
+
*/
|
|
72
|
+
export declare function parseMongoReplicationStream({ batches, ...options }: MongoReplicationParserOptions & {
|
|
73
|
+
batches: AsyncIterable<RawMongoReplicationStreamBatch>;
|
|
74
|
+
}): AsyncGenerator<MongoReplicationStreamBatch>;
|
|
@@ -1,11 +1,106 @@
|
|
|
1
1
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
2
|
import { buildMongoChangeStreamPipeline } from './MongoReplicationQueryProvider.js';
|
|
3
|
+
import { MongoReplicationFilterResult } from './MongoReplicationStreamTypes.js';
|
|
3
4
|
import { parseChangeDocument, rawChangeStream } from './RawChangeStream.js';
|
|
5
|
+
import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
6
|
+
/**
|
|
7
|
+
* Wraps changes with a single-use iterator. Caches the token and metadata after successful exhaustion.
|
|
8
|
+
*/
|
|
9
|
+
export function createMongoReplicationStreamBatch({ events, resolveResumeToken, resolveMetadata = () => ({ filteredCount: null }) }) {
|
|
10
|
+
let started = false;
|
|
11
|
+
let complete = false;
|
|
12
|
+
let resumeToken;
|
|
13
|
+
let metadata;
|
|
14
|
+
function assertComplete() {
|
|
15
|
+
// Progress is only safe to read after every event has been consumed and finalization succeeds.
|
|
16
|
+
if (!complete) {
|
|
17
|
+
throw new ReplicationAssertionError('Consume all batch events before requesting token or metadata');
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return {
|
|
21
|
+
events: {
|
|
22
|
+
[Symbol.iterator]() {
|
|
23
|
+
// A second pass would repeat parsing/filtering against state that has already advanced.
|
|
24
|
+
if (started) {
|
|
25
|
+
throw new ReplicationAssertionError('Batch events can only be consumed once');
|
|
26
|
+
}
|
|
27
|
+
started = true;
|
|
28
|
+
return iterate();
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
getResumeToken() {
|
|
32
|
+
assertComplete();
|
|
33
|
+
return resumeToken;
|
|
34
|
+
},
|
|
35
|
+
get metadata() {
|
|
36
|
+
assertComplete();
|
|
37
|
+
return metadata;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
function* iterate() {
|
|
41
|
+
// Delegate lazily; an error or early return skips the finalization below.
|
|
42
|
+
yield* events;
|
|
43
|
+
// Cache this batch's final results before later batches can change the parser or filter state.
|
|
44
|
+
resumeToken = resolveResumeToken();
|
|
45
|
+
metadata = Object.freeze({ ...resolveMetadata() });
|
|
46
|
+
complete = true;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Filter complete data changes while retaining source batch boundaries and recovery bookkeeping.
|
|
51
|
+
* Return MongoReplicationFilterResult.KEEP to forward a change, EXCLUDE to omit it, or DELETE for an exit.
|
|
52
|
+
* Synthetic deletes retain the source identity, resume token and transaction metadata.
|
|
53
|
+
* The callback runs synchronously and lazily; failures and early exits close the underlying stream.
|
|
54
|
+
*/
|
|
55
|
+
export async function* filterMongoReplicationStream({ batches, filter }) {
|
|
56
|
+
for await (const sourceBatch of batches) {
|
|
57
|
+
let filteredCount = 0;
|
|
58
|
+
const batch = createMongoReplicationStreamBatch({
|
|
59
|
+
events: filterEvents(),
|
|
60
|
+
resolveResumeToken: () => sourceBatch.getResumeToken(),
|
|
61
|
+
resolveMetadata: () => ({ filteredCount: (sourceBatch.metadata.filteredCount ?? 0) + filteredCount })
|
|
62
|
+
});
|
|
63
|
+
yield batch;
|
|
64
|
+
// Assert that the consumer exhausted this batch before requesting the next one, even if every event
|
|
65
|
+
// was filtered out. The token value is unused; getResumeToken() throws if consumption is incomplete.
|
|
66
|
+
batch.getResumeToken();
|
|
67
|
+
function* filterEvents() {
|
|
68
|
+
for (const change of sourceBatch.events) {
|
|
69
|
+
const { event } = change;
|
|
70
|
+
if ((event.operationType != 'insert' &&
|
|
71
|
+
event.operationType != 'update' &&
|
|
72
|
+
event.operationType != 'replace' &&
|
|
73
|
+
event.operationType != 'delete') ||
|
|
74
|
+
event.ns.coll == CHECKPOINTS_COLLECTION) {
|
|
75
|
+
yield change;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
const result = filter(event);
|
|
79
|
+
if (result == MongoReplicationFilterResult.EXCLUDE) {
|
|
80
|
+
filteredCount++;
|
|
81
|
+
}
|
|
82
|
+
else if (result == MongoReplicationFilterResult.DELETE) {
|
|
83
|
+
yield { ...change, event: { ...event, operationType: 'delete' } };
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
yield change;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
4
92
|
/**
|
|
5
93
|
* Shared opening path for ongoing replication, snapshot barriers and resume validation.
|
|
6
94
|
*/
|
|
7
95
|
export function openMongoReplicationStream({ db, queryProvider, namespaceFilter, isDocumentDb, usePostImages, position, skipInitialTimestamp = false, options, onBatch }) {
|
|
8
|
-
const
|
|
96
|
+
const parsing = {
|
|
97
|
+
defaultSchema: db.databaseName,
|
|
98
|
+
multipleDatabases: namespaceFilter.multipleDatabases,
|
|
99
|
+
startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
|
|
100
|
+
signal: options.signal,
|
|
101
|
+
logger: options.logger
|
|
102
|
+
};
|
|
103
|
+
const openRaw = async function* ({ pipelineStages = [], imageOptions = {} }) {
|
|
9
104
|
const clusterScope = isDocumentDb || namespaceFilter.multipleDatabases;
|
|
10
105
|
const streamOptions = {
|
|
11
106
|
...imageOptions,
|
|
@@ -27,31 +122,27 @@ export function openMongoReplicationStream({ db, queryProvider, namespaceFilter,
|
|
|
27
122
|
pipelineStages,
|
|
28
123
|
includeSplitLargeEvent: !isDocumentDb
|
|
29
124
|
});
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
// still be delivered. Timestamp deduplication applies only to legacy positions without tokens.
|
|
36
|
-
startAfter: skipInitialTimestamp && position?.resumeAfter == null ? position?.startAfter : undefined,
|
|
37
|
-
signal: options.signal,
|
|
38
|
-
logger: options.logger,
|
|
39
|
-
onBatch
|
|
40
|
-
});
|
|
125
|
+
for await (const batch of rawChangeStream(clusterScope ? db.client.db('admin') : db, pipeline, options)) {
|
|
126
|
+
options.signal?.throwIfAborted();
|
|
127
|
+
using batchScope = onBatch?.(batch) || undefined;
|
|
128
|
+
yield { events: batch.events, resumeToken: batch.resumeToken };
|
|
129
|
+
}
|
|
41
130
|
};
|
|
42
|
-
return queryProvider.openChangeStream({
|
|
131
|
+
return queryProvider.openChangeStream({
|
|
132
|
+
open: (options) => parseMongoReplicationStream({ batches: openRaw(options), ...parsing })
|
|
133
|
+
});
|
|
43
134
|
}
|
|
44
135
|
/**
|
|
45
|
-
*
|
|
136
|
+
* Creates a stateful parser for one raw MongoDB stream. Parses BSON entries, joins split-event
|
|
137
|
+
* fragments across batches, normalizes namespaces and skips events covered by legacy timestamps.
|
|
138
|
+
* After processing a batch, callers can request safe progress. Check completion on natural EOF,
|
|
139
|
+
* not cancellation.
|
|
46
140
|
*/
|
|
47
|
-
export
|
|
141
|
+
export function createMongoReplicationStreamParser({ defaultSchema, multipleDatabases, startAfter, signal, logger }) {
|
|
48
142
|
let splitDocument = null;
|
|
49
143
|
let flexDbNameWorkaroundLogged = false;
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
// The optional scope measures batch processing, excluding the next network wait.
|
|
53
|
-
using batchScope = onBatch?.(batch) || undefined;
|
|
54
|
-
for (const [index, rawEvent] of batch.events.entries()) {
|
|
144
|
+
return {
|
|
145
|
+
parse(rawEvent) {
|
|
55
146
|
signal?.throwIfAborted();
|
|
56
147
|
let event = parseChangeDocument(rawEvent);
|
|
57
148
|
const fragment = event.splitEvent;
|
|
@@ -65,16 +156,18 @@ export async function* readMongoReplicationStream({ batches, defaultSchema, mult
|
|
|
65
156
|
throw new ReplicationAssertionError(`Unexpected splitEvent: ${JSON.stringify(fragment)}`);
|
|
66
157
|
}
|
|
67
158
|
splitDocument = splitDocument == null ? event : Object.assign(splitDocument, event);
|
|
68
|
-
if (fragment.fragment != fragment.of)
|
|
69
|
-
|
|
159
|
+
if (fragment.fragment != fragment.of) {
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
70
162
|
event = splitDocument;
|
|
71
163
|
splitDocument = null;
|
|
72
164
|
}
|
|
73
165
|
else if (splitDocument != null) {
|
|
74
166
|
throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
|
|
75
167
|
}
|
|
76
|
-
if (startAfter != null && event.clusterTime?.lte(startAfter))
|
|
77
|
-
|
|
168
|
+
if (startAfter != null && event.clusterTime?.lte(startAfter)) {
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
78
171
|
if (!multipleDatabases &&
|
|
79
172
|
'ns' in event &&
|
|
80
173
|
event.ns.db != defaultSchema &&
|
|
@@ -88,14 +181,48 @@ export async function* readMongoReplicationStream({ batches, defaultSchema, mult
|
|
|
88
181
|
logger?.warn(`Incorrect DB name in change stream: ${originalDatabase}. Changed to ${defaultSchema}.`);
|
|
89
182
|
}
|
|
90
183
|
}
|
|
91
|
-
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
|
|
184
|
+
return event;
|
|
185
|
+
},
|
|
186
|
+
/**
|
|
187
|
+
* Returns the given transport token if it is safe to persist, or undefined while a split event is unfinished.
|
|
188
|
+
*/
|
|
189
|
+
getSafeResumeToken(resumeToken) {
|
|
190
|
+
return splitDocument == null ? resumeToken : undefined;
|
|
191
|
+
},
|
|
192
|
+
assertComplete() {
|
|
193
|
+
if (splitDocument != null) {
|
|
194
|
+
throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
|
|
195
|
+
}
|
|
95
196
|
}
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Wrap raw batches in lazy event iterables. BSON parsing and split-event assembly happen
|
|
201
|
+
* as consumers request events, without building a parsed array. After exhaustion, each batch retains
|
|
202
|
+
* its own resume-token result even when later batches change the parser state.
|
|
203
|
+
* Consume each batch completely before requesting the next; returning early closes the source.
|
|
204
|
+
*/
|
|
205
|
+
export async function* parseMongoReplicationStream({ batches, ...options }) {
|
|
206
|
+
const parser = createMongoReplicationStreamParser(options);
|
|
207
|
+
for await (const sourceBatch of batches) {
|
|
208
|
+
options.signal?.throwIfAborted();
|
|
209
|
+
const batch = createMongoReplicationStreamBatch({
|
|
210
|
+
events: parseEvents(sourceBatch.events),
|
|
211
|
+
resolveResumeToken: () => parser.getSafeResumeToken(sourceBatch.resumeToken)
|
|
212
|
+
});
|
|
213
|
+
yield batch;
|
|
214
|
+
// Split-event state is shared across batches, so the consumer must finish this one before the next is read.
|
|
215
|
+
// This also covers empty batches.
|
|
216
|
+
batch.getResumeToken();
|
|
96
217
|
}
|
|
97
|
-
|
|
98
|
-
|
|
218
|
+
parser.assertComplete();
|
|
219
|
+
function* parseEvents(rawEvents) {
|
|
220
|
+
for (let index = 0; index < rawEvents.length; index++) {
|
|
221
|
+
const event = parser.parse(rawEvents[index]);
|
|
222
|
+
if (event != null) {
|
|
223
|
+
yield { event, hasBufferedChanges: index < rawEvents.length - 1 };
|
|
224
|
+
}
|
|
225
|
+
}
|
|
99
226
|
}
|
|
100
227
|
}
|
|
101
228
|
//# sourceMappingURL=MongoReplicationStream.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoReplicationStream.js","sourceRoot":"","sources":["../../src/replication/MongoReplicationStream.ts"],"names":[],"mappings":"AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"MongoReplicationStream.js","sourceRoot":"","sources":["../../src/replication/MongoReplicationStream.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAE,8BAA8B,EAAiC,MAAM,oCAAoC,CAAC;AACnH,OAAO,EAEL,4BAA4B,EAO7B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAEL,mBAAmB,EAEnB,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE;;GAEG;AACH,MAAM,UAAU,iCAAiC,CAAC,EAChD,MAAM,EACN,kBAAkB,EAClB,eAAe,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAclD;IACC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,WAA0C,CAAC;IAC/C,IAAI,QAA6C,CAAC;IAClD,SAAS,cAAc;QACrB,+FAA+F;QAC/F,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,yBAAyB,CAAC,8DAA8D,CAAC,CAAC;QACtG,CAAC;IACH,CAAC;IACD,OAAO;QACL,MAAM,EAAE;YACN,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACf,wFAAwF;gBACxF,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;gBAChF,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,OAAO,EAAE,CAAC;YACnB,CAAC;SACF;QACD,cAAc;YACZ,cAAc,EAAE,CAAC;YACjB,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,QAAQ;YACV,cAAc,EAAE,CAAC;YACjB,OAAO,QAAQ,CAAC;QAClB,CAAC;KACF,CAAC;IAEF,QAAQ,CAAC,CAAC,OAAO;QACf,0EAA0E;QAC1E,KAAK,CAAC,CAAC,MAAM,CAAC;QACd,+FAA+F;QAC/F,WAAW,GAAG,kBAAkB,EAAE,CAAC;QACnC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,eAAe,EAAE,EAAE,CAAC,CAAC;QACnD,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,4BAA4B,CAAC,EAClD,OAAO,EACP,MAAM,EACwB;IAC9B,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,OAAO,EAAE,CAAC;QACxC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,KAAK,GAAG,iCAAiC,CAAC;YAC9C,MAAM,EAAE,YAAY,EAAE;YACtB,kBAAkB,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,cAAc,EAAE;YACtD,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,aAAa,EAAE,CAAC;SACtG,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;QACZ,oGAAoG;QACpG,qGAAqG;QACrG,KAAK,CAAC,cAAc,EAAE,CAAC;QAEvB,QAAQ,CAAC,CAAC,YAAY;YACpB,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;gBACzB,IACE,CAAC,KAAK,CAAC,aAAa,IAAI,QAAQ;oBAC9B,KAAK,CAAC,aAAa,IAAI,QAAQ;oBAC/B,KAAK,CAAC,aAAa,IAAI,SAAS;oBAChC,KAAK,CAAC,aAAa,IAAI,QAAQ,CAAC;oBAClC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,sBAAsB,EACvC,CAAC;oBACD,MAAM,MAAM,CAAC;oBACb,SAAS;gBACX,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC7B,IAAI,MAAM,IAAI,4BAA4B,CAAC,OAAO,EAAE,CAAC;oBACnD,aAAa,EAAE,CAAC;gBAClB,CAAC;qBAAM,IAAI,MAAM,IAAI,4BAA4B,CAAC,MAAM,EAAE,CAAC;oBACzD,MAAM,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,CAAC;gBACpE,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC;gBACf,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,EACzC,EAAE,EACF,aAAa,EACb,eAAe,EACf,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,oBAAoB,GAAG,KAAK,EAC5B,OAAO,EACP,OAAO,EAcR;IACC,MAAM,OAAO,GAAG;QACd,aAAa,EAAE,EAAE,CAAC,YAAY;QAC9B,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;QACpD,UAAU,EAAE,oBAAoB,IAAI,QAAQ,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS;QACpG,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,SAAS,CAAC,EAAE,EAC/B,cAAc,GAAG,EAAE,EACnB,YAAY,GAAG,EAAE,EACa;QAC9B,MAAM,YAAY,GAAG,YAAY,IAAI,eAAe,CAAC,iBAAiB,CAAC;QACvE,MAAM,aAAa,GAAmB;YACpC,GAAG,YAAY;YACf,YAAY,EAAE,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc;YAC1E,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC;QACF,mGAAmG;QACnG,4DAA4D;QAC5D,IAAI,QAAQ,EAAE,WAAW,IAAI,IAAI,EAAE,CAAC;YAClC,aAAa,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;QACnD,CAAC;aAAM,IAAI,QAAQ,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC;YACxC,aAAa,CAAC,oBAAoB,GAAG,QAAQ,CAAC,UAAU,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAG,8BAA8B,CAAC;YAC9C,aAAa;YACb,cAAc,EAAE,eAAe,CAAC,MAAM;YACtC,cAAc;YACd,sBAAsB,EAAE,CAAC,YAAY;SACtC,CAAC,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,eAAe,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;YACxG,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC;YACjD,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QACjE,CAAC;IACH,CAAC,CAAC;IAEF,OAAO,aAAa,CAAC,gBAAgB,CAAC;QACpC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC;KAC1F,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kCAAkC,CAAC,EACjD,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,MAAM,EACN,MAAM,EACwB;IAC9B,IAAI,aAAa,GAAyC,IAAI,CAAC;IAC/D,IAAI,0BAA0B,GAAG,KAAK,CAAC;IACvC,OAAO;QACL,KAAK,CAAC,QAAgB;YACpB,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,IAAI,KAAK,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC;YAClC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,gGAAgG;gBAChG,4FAA4F;gBAC5F,MAAM,gBAAgB,GAAG,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,UAAW,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC5F,IACE,QAAQ,CAAC,QAAQ,IAAI,gBAAgB;oBACrC,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC,EAAE;oBAC/B,CAAC,aAAa,IAAI,IAAI,IAAI,QAAQ,CAAC,EAAE,IAAI,aAAa,CAAC,UAAW,CAAC,EAAE,CAAC,EACtE,CAAC;oBACD,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAC5F,CAAC;gBACD,aAAa,GAAG,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;gBACpF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACrC,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,KAAK,GAAG,aAAa,CAAC;gBACtB,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;gBACjC,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC5G,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7D,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IACE,CAAC,iBAAiB;gBAClB,IAAI,IAAI,KAAK;gBACb,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,aAAa;gBAC5B,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC,EACzC,CAAC;gBACD,2FAA2F;gBAC3F,4FAA4F;gBAC5F,MAAM,gBAAgB,GAAG,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;gBACrC,KAAK,CAAC,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC;gBAC5B,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBAChC,0BAA0B,GAAG,IAAI,CAAC;oBAClC,MAAM,EAAE,IAAI,CAAC,uCAAuC,gBAAgB,gBAAgB,aAAa,GAAG,CAAC,CAAC;gBACxG,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QACD;;WAEG;QACH,kBAAkB,CAAC,WAA8B;YAC/C,OAAO,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,CAAC;QACD,cAAc;YACZ,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,yBAAyB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC5G,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,2BAA2B,CAAC,EACjD,OAAO,EACP,GAAG,OAAO,EAGX;IACC,MAAM,MAAM,GAAG,kCAAkC,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,OAAO,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,iCAAiC,CAAC;YAC9C,MAAM,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC;YACvC,kBAAkB,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,WAAW,CAAC,WAAW,CAAC;SAC7E,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;QACZ,4GAA4G;QAC5G,kCAAkC;QAClC,KAAK,CAAC,cAAc,EAAE,CAAC;IACzB,CAAC;IACD,MAAM,CAAC,cAAc,EAAE,CAAC;IAExB,QAAQ,CAAC,CAAC,WAAW,CAAC,SAAmB;QACvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpE,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import type { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import type { Logger } from '@powersync/lib-services-framework';
|
|
3
|
+
import type { ProjectedChangeStreamDocument } from './RawChangeStream.js';
|
|
4
|
+
/**
|
|
5
|
+
* An unparsed MongoDB response batch. Consumers parse its entries incrementally and retain unfinished
|
|
6
|
+
* split events across batches before deciding whether its resume token is safe to persist.
|
|
7
|
+
*/
|
|
8
|
+
export interface RawMongoReplicationStreamBatch {
|
|
9
|
+
/**
|
|
10
|
+
* Original BSON entries in source order, including possible split-event fragments.
|
|
11
|
+
*/
|
|
12
|
+
events: Buffer[];
|
|
13
|
+
/**
|
|
14
|
+
* Transport position after this batch. It may cover an incomplete event and is not yet safe progress.
|
|
15
|
+
*/
|
|
16
|
+
resumeToken: mongo.ResumeToken;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* A complete event and its position within the original source batch.
|
|
20
|
+
*/
|
|
21
|
+
export interface MongoReplicationStreamChange {
|
|
22
|
+
event: ProjectedChangeStreamDocument;
|
|
23
|
+
/**
|
|
24
|
+
* MongoDB returned more entries after this event, even if a provider later excludes them or the
|
|
25
|
+
* parser retains them as unfinished split fragments. For [checkpoint, excluded change], preserve
|
|
26
|
+
* true on the checkpoint so ChangeStream groups commits instead of treating it as caught up.
|
|
27
|
+
* Providers must preserve this value when forwarding or transforming events.
|
|
28
|
+
*/
|
|
29
|
+
hasBufferedChanges: boolean;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Statistics for one fully consumed batch, independent of whether its resume token is safe.
|
|
33
|
+
*/
|
|
34
|
+
export interface MongoReplicationStreamBatchMetadata {
|
|
35
|
+
/**
|
|
36
|
+
* Complete events excluded from this batch. Null means no filtering was applied; zero means
|
|
37
|
+
* filtering was applied but no events were excluded. Counts do not carry across batches.
|
|
38
|
+
*/
|
|
39
|
+
readonly filteredCount: number | null;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* A single-use batch of complete events, parsed or filtered lazily as the consumer iterates.
|
|
43
|
+
* Exhaust events before reading its resume token, metadata or the next batch.
|
|
44
|
+
*/
|
|
45
|
+
export interface MongoReplicationStreamBatch {
|
|
46
|
+
/**
|
|
47
|
+
* Complete changes in source order, parsed or filtered lazily during synchronous iteration.
|
|
48
|
+
* Consume once and fully. It may be empty after filtering or while assembling a split event.
|
|
49
|
+
*/
|
|
50
|
+
events: Iterable<MongoReplicationStreamChange>;
|
|
51
|
+
/**
|
|
52
|
+
* Returns the safe resume token after events are fully consumed, or undefined while a split event
|
|
53
|
+
* is unfinished. Throws before exhaustion. The result remains stable after later batches.
|
|
54
|
+
*/
|
|
55
|
+
getResumeToken(): mongo.ResumeToken | undefined;
|
|
56
|
+
/**
|
|
57
|
+
* Final statistics for this batch, available even without a safe resume token.
|
|
58
|
+
* Throws before events are fully consumed; the result is immutable and cached afterward.
|
|
59
|
+
*/
|
|
60
|
+
readonly metadata: MongoReplicationStreamBatchMetadata;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Provider-controlled additions to the shared cursor setup. Source scope, resume position, post-image
|
|
64
|
+
* requirements and cancellation remain controlled by the MongoDB replication module.
|
|
65
|
+
*/
|
|
66
|
+
export interface MongoReplicationStreamOptions {
|
|
67
|
+
/**
|
|
68
|
+
* Inserted after namespace selection and before the final large-event split stage.
|
|
69
|
+
*/
|
|
70
|
+
pipelineStages?: mongo.Document[];
|
|
71
|
+
/**
|
|
72
|
+
* Request pre-images without overriding the reader's post-image, resume or scope options.
|
|
73
|
+
*/
|
|
74
|
+
imageOptions?: Pick<mongo.ChangeStreamOptions, 'fullDocumentBeforeChange'>;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Bound source opener supplied to a query provider for one stream invocation.
|
|
78
|
+
* Providers may wrap the lazy event iterables to filter or transform complete changes.
|
|
79
|
+
*/
|
|
80
|
+
export interface MongoReplicationStreamContext {
|
|
81
|
+
/**
|
|
82
|
+
* Open batches whose events are parsed lazily, with cursor scope, position, cancellation and
|
|
83
|
+
* accounting already bound. Each call owns independent split-event assembly and parser state.
|
|
84
|
+
* Events are complete; progress is withheld while a split event is unfinished.
|
|
85
|
+
* Close the iterator on completion, cancellation or failure (for await handles this).
|
|
86
|
+
*/
|
|
87
|
+
open(options: MongoReplicationStreamOptions): AsyncIterableIterator<MongoReplicationStreamBatch>;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Source context for lazy parsing of raw BSON events in default and filtering providers.
|
|
91
|
+
* These options control namespace normalization, legacy timestamp skipping and cancellation; split-event
|
|
92
|
+
* state belongs to each parser instance and must not be shared between independently opened streams.
|
|
93
|
+
*/
|
|
94
|
+
export interface MongoReplicationParserOptions {
|
|
95
|
+
/**
|
|
96
|
+
* Expected database name when correcting Atlas Flex's prefixed namespaces in single-database mode.
|
|
97
|
+
*/
|
|
98
|
+
defaultSchema: string;
|
|
99
|
+
/**
|
|
100
|
+
* Preserve database names when the stream covers multiple databases instead of normalizing them.
|
|
101
|
+
*/
|
|
102
|
+
multipleDatabases: boolean;
|
|
103
|
+
/**
|
|
104
|
+
* Exclude complete events at or before this legacy resume timestamp. Leave unset for exact-token
|
|
105
|
+
* resumes, where later events can share the same timestamp, and for snapshot barrier probes.
|
|
106
|
+
*/
|
|
107
|
+
startAfter?: mongo.Timestamp | null;
|
|
108
|
+
/**
|
|
109
|
+
* Abort parsing when the owning replication attempt is cancelled. Resolving a batch's progress is not
|
|
110
|
+
* interrupted, so consumers must check the signal themselves before persisting a token.
|
|
111
|
+
*/
|
|
112
|
+
signal?: AbortSignal;
|
|
113
|
+
/**
|
|
114
|
+
* Receives the namespace-correction warning, at most once per parser instance.
|
|
115
|
+
*/
|
|
116
|
+
logger?: Logger;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* A complete data change eligible for filtering. Checkpoints and DDL bypass the filter.
|
|
120
|
+
*/
|
|
121
|
+
export type MongoReplicationDataChange = Extract<ProjectedChangeStreamDocument, {
|
|
122
|
+
operationType: 'insert' | 'update' | 'replace' | 'delete';
|
|
123
|
+
}>;
|
|
124
|
+
/**
|
|
125
|
+
* The action to apply to a complete data change after evaluating filter membership.
|
|
126
|
+
*/
|
|
127
|
+
export declare enum MongoReplicationFilterResult {
|
|
128
|
+
/**
|
|
129
|
+
* Forward the original change.
|
|
130
|
+
*/
|
|
131
|
+
KEEP = "keep",
|
|
132
|
+
/**
|
|
133
|
+
* Omit the change while retaining its batch's recovery progress.
|
|
134
|
+
*/
|
|
135
|
+
EXCLUDE = "exclude",
|
|
136
|
+
/**
|
|
137
|
+
* Emit a delete because the document left the filtered set.
|
|
138
|
+
*/
|
|
139
|
+
DELETE = "delete"
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Synchronously decide how to replicate a complete data change. Checkpoints and DDL bypass this callback.
|
|
143
|
+
*/
|
|
144
|
+
export type MongoReplicationFilter = (event: MongoReplicationDataChange) => MongoReplicationFilterResult;
|
|
145
|
+
/**
|
|
146
|
+
* Source batches and membership decisions for the shared replication filter.
|
|
147
|
+
*/
|
|
148
|
+
export interface MongoReplicationFilterOptions {
|
|
149
|
+
/**
|
|
150
|
+
* Ordered source batches whose boundaries and recovery progress must be preserved.
|
|
151
|
+
*/
|
|
152
|
+
batches: AsyncIterable<MongoReplicationStreamBatch>;
|
|
153
|
+
/**
|
|
154
|
+
* Decide whether to keep, exclude or delete each complete data change.
|
|
155
|
+
*/
|
|
156
|
+
filter: MongoReplicationFilter;
|
|
157
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The action to apply to a complete data change after evaluating filter membership.
|
|
3
|
+
*/
|
|
4
|
+
export var MongoReplicationFilterResult;
|
|
5
|
+
(function (MongoReplicationFilterResult) {
|
|
6
|
+
/**
|
|
7
|
+
* Forward the original change.
|
|
8
|
+
*/
|
|
9
|
+
MongoReplicationFilterResult["KEEP"] = "keep";
|
|
10
|
+
/**
|
|
11
|
+
* Omit the change while retaining its batch's recovery progress.
|
|
12
|
+
*/
|
|
13
|
+
MongoReplicationFilterResult["EXCLUDE"] = "exclude";
|
|
14
|
+
/**
|
|
15
|
+
* Emit a delete because the document left the filtered set.
|
|
16
|
+
*/
|
|
17
|
+
MongoReplicationFilterResult["DELETE"] = "delete";
|
|
18
|
+
})(MongoReplicationFilterResult || (MongoReplicationFilterResult = {}));
|
|
19
|
+
//# sourceMappingURL=MongoReplicationStreamTypes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MongoReplicationStreamTypes.js","sourceRoot":"","sources":["../../src/replication/MongoReplicationStreamTypes.ts"],"names":[],"mappings":"AAqIA;;GAEG;AACH,MAAM,CAAN,IAAY,4BAaX;AAbD,WAAY,4BAA4B;IACtC;;OAEG;IACH,6CAAa,CAAA;IACb;;OAEG;IACH,mDAAmB,CAAA;IACnB;;OAEG;IACH,iDAAiB,CAAA;AACnB,CAAC,EAbW,4BAA4B,KAA5B,4BAA4B,QAavC"}
|