@powersync/service-module-mongodb 0.0.0-dev-20261007110800 → 0.0.0-dev-20261015132441
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -11
- package/benchmarks/env.ts +11 -0
- package/benchmarks/replication_branch_throughput.benchmark.ts +240 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +5 -0
- package/dist/api/MongoRouteAPIAdapter.js +11 -4
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/module/MongoModule.js +1 -0
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +2 -1
- package/dist/replication/ChangeStream.js +164 -150
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +17 -9
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/MongoReplicationQueryProvider.d.ts +9 -11
- package/dist/replication/MongoReplicationQueryProvider.js.map +1 -1
- package/dist/replication/MongoReplicationStream.d.ts +37 -48
- package/dist/replication/MongoReplicationStream.js +157 -30
- package/dist/replication/MongoReplicationStream.js.map +1 -1
- package/dist/replication/MongoReplicationStreamTypes.d.ts +157 -0
- package/dist/replication/MongoReplicationStreamTypes.js +19 -0
- package/dist/replication/MongoReplicationStreamTypes.js.map +1 -0
- package/dist/replication/MongoSnapshotter.js +59 -30
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/estimateFilteredSnapshotCount.d.ts +12 -0
- package/dist/replication/estimateFilteredSnapshotCount.js +79 -0
- package/dist/replication/estimateFilteredSnapshotCount.js.map +1 -0
- package/dist/replication/replication-index.d.ts +1 -0
- package/dist/replication/replication-index.js +1 -0
- package/dist/replication/replication-index.js.map +1 -1
- package/package.json +11 -10
- package/src/api/MongoRouteAPIAdapter.ts +12 -5
- package/src/module/MongoModule.ts +1 -0
- package/src/replication/ChangeStream.ts +183 -173
- package/src/replication/ChangeStreamReplicationJob.ts +16 -9
- package/src/replication/MongoReplicationQueryProvider.ts +9 -11
- package/src/replication/MongoReplicationStream.ts +196 -83
- package/src/replication/MongoReplicationStreamTypes.ts +169 -0
- package/src/replication/MongoSnapshotter.ts +64 -33
- package/src/replication/estimateFilteredSnapshotCount.ts +93 -0
- package/src/replication/replication-index.ts +1 -0
- package/test/src/change_event_memory.test.ts +33 -0
- package/test/src/change_stream_abort.test.ts +110 -0
- package/test/src/filtering_example.test.ts +65 -0
- package/test/src/filtering_lifecycle.test.ts +101 -0
- package/test/src/query_provider_availability.test.ts +1 -0
- package/test/src/query_provider_diagnostics.test.ts +55 -0
- package/test/src/replication_stream.test.ts +411 -34
- package/test/src/snapshot_count.test.ts +146 -0
- package/test/src/snapshot_count_integration.test.ts +70 -0
- package/test/src/snapshot_query_provider.test.ts +21 -2
- package/test/src/stream_progress.test.ts +62 -44
- package/test/tsconfig.json +2 -2
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.benchmark.config.ts +15 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { estimateFilteredSnapshotCount } from '@module/replication/estimateFilteredSnapshotCount.js';
|
|
2
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
|
+
import { createLogger } from '@powersync/lib-services-framework';
|
|
4
|
+
import { afterEach, describe, expect, test, vi } from 'vitest';
|
|
5
|
+
|
|
6
|
+
const FILTER = { $expr: { $eq: ['$label', 'live'] } };
|
|
7
|
+
const LOGGER = createLogger({ silent: true });
|
|
8
|
+
|
|
9
|
+
function fixture() {
|
|
10
|
+
const client = new mongo.MongoClient('mongodb://localhost:27017');
|
|
11
|
+
const collection = client.db('snapshot_count_unit').collection('documents');
|
|
12
|
+
const controller = new AbortController();
|
|
13
|
+
const count = vi.spyOn(collection, 'countDocuments').mockResolvedValue(42);
|
|
14
|
+
const metadata = vi.spyOn(collection, 'estimatedDocumentCount').mockResolvedValue(100_000);
|
|
15
|
+
const cursor = collection.aggregate();
|
|
16
|
+
const next = vi.spyOn(cursor, 'next').mockResolvedValue({ sampled: [{ count: 1_000 }], matches: [{ count: 250 }] });
|
|
17
|
+
const close = vi.spyOn(cursor, 'close').mockResolvedValue();
|
|
18
|
+
const aggregate = vi.spyOn(collection, 'aggregate').mockReturnValue(cursor);
|
|
19
|
+
const estimate = () =>
|
|
20
|
+
estimateFilteredSnapshotCount({ collection, filter: FILTER, signal: controller.signal, logger: LOGGER });
|
|
21
|
+
return { controller, count, metadata, next, close, aggregate, estimate };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
vi.restoreAllMocks();
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
describe('filtered snapshot count', () => {
|
|
29
|
+
test('uses a bounded exact count with the snapshot collation', async () => {
|
|
30
|
+
const context = fixture();
|
|
31
|
+
expect(await context.estimate()).toBe(42);
|
|
32
|
+
expect(context.count).toHaveBeenCalledWith(FILTER, {
|
|
33
|
+
collation: { locale: 'simple' },
|
|
34
|
+
readConcern: 'majority',
|
|
35
|
+
timeoutMS: 10_000,
|
|
36
|
+
signal: context.controller.signal
|
|
37
|
+
});
|
|
38
|
+
expect(context.metadata).not.toHaveBeenCalled();
|
|
39
|
+
expect(context.aggregate).not.toHaveBeenCalled();
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('preserves an exact zero without sampling', async () => {
|
|
43
|
+
const context = fixture();
|
|
44
|
+
context.count.mockResolvedValue(0);
|
|
45
|
+
expect(await context.estimate()).toBe(0);
|
|
46
|
+
expect(context.metadata).not.toHaveBeenCalled();
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('samples before filtering and scales by collection metadata after a count timeout', async () => {
|
|
50
|
+
const context = fixture();
|
|
51
|
+
context.count.mockRejectedValue(new mongo.MongoOperationTimeoutError('count timed out'));
|
|
52
|
+
expect(await context.estimate()).toBe(25_000);
|
|
53
|
+
expect(context.aggregate).toHaveBeenCalledWith(
|
|
54
|
+
[
|
|
55
|
+
{ $sample: { size: 1_000 } },
|
|
56
|
+
{ $facet: { sampled: [{ $count: 'count' }], matches: [{ $match: FILTER }, { $count: 'count' }] } }
|
|
57
|
+
],
|
|
58
|
+
expect.objectContaining({
|
|
59
|
+
collation: { locale: 'simple' },
|
|
60
|
+
readConcern: 'majority',
|
|
61
|
+
signal: context.controller.signal,
|
|
62
|
+
allowDiskUse: false
|
|
63
|
+
})
|
|
64
|
+
);
|
|
65
|
+
const timeoutMS = context.aggregate.mock.calls[0][1]!.timeoutMS!;
|
|
66
|
+
expect(timeoutMS).toBeGreaterThan(0);
|
|
67
|
+
expect(timeoutMS).toBeLessThanOrEqual(10_000);
|
|
68
|
+
expect(context.close).toHaveBeenCalled();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test.each([
|
|
72
|
+
[10, 1],
|
|
73
|
+
[100_000_000, 10_000]
|
|
74
|
+
])('bounds the sample size for a collection of %s documents', async (total, sampleSize) => {
|
|
75
|
+
const context = fixture();
|
|
76
|
+
context.count.mockRejectedValue(new Error('count failed'));
|
|
77
|
+
context.metadata.mockResolvedValue(total);
|
|
78
|
+
await context.estimate();
|
|
79
|
+
expect(context.aggregate.mock.calls[0][0]![0]).toEqual({ $sample: { size: sampleSize } });
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test('uses the actual number sampled when metadata or collection size has changed', async () => {
|
|
83
|
+
const context = fixture();
|
|
84
|
+
context.count.mockRejectedValue(new Error('count failed'));
|
|
85
|
+
context.next.mockResolvedValue({ sampled: [{ count: 80 }], matches: [{ count: 20 }] });
|
|
86
|
+
expect(await context.estimate()).toBe(25_000);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test.each([null, { sampled: [], matches: [] }, { sampled: [{ count: 1_000 }], matches: [] }])(
|
|
90
|
+
'leaves the total unknown for an empty sample or no matches: %j',
|
|
91
|
+
async (sample) => {
|
|
92
|
+
const context = fixture();
|
|
93
|
+
context.count.mockRejectedValue(new Error('count failed'));
|
|
94
|
+
context.next.mockResolvedValue(sample);
|
|
95
|
+
expect(await context.estimate()).toBe(-1);
|
|
96
|
+
expect(context.close).toHaveBeenCalled();
|
|
97
|
+
}
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
test.each(['metadata', 'sample', 'empty metadata', 'budget exhausted'])(
|
|
101
|
+
'leaves the total unknown after %s',
|
|
102
|
+
async (failure) => {
|
|
103
|
+
const context = fixture();
|
|
104
|
+
context.count.mockRejectedValue(new Error('count failed'));
|
|
105
|
+
if (failure == 'metadata') {
|
|
106
|
+
context.metadata.mockRejectedValue(new Error('metadata failed'));
|
|
107
|
+
} else if (failure == 'sample') {
|
|
108
|
+
context.next.mockRejectedValue(new mongo.MongoOperationTimeoutError('sample timed out'));
|
|
109
|
+
} else if (failure == 'empty metadata') {
|
|
110
|
+
context.metadata.mockResolvedValue(0);
|
|
111
|
+
} else {
|
|
112
|
+
const { performance } = await import('node:perf_hooks');
|
|
113
|
+
vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValue(10_001);
|
|
114
|
+
}
|
|
115
|
+
expect(await context.estimate()).toBe(-1);
|
|
116
|
+
if (failure == 'sample') {
|
|
117
|
+
expect(context.close).toHaveBeenCalled();
|
|
118
|
+
} else {
|
|
119
|
+
expect(context.aggregate).not.toHaveBeenCalled();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
);
|
|
123
|
+
|
|
124
|
+
test.each(['before count', 'during count', 'during sample'])('propagates cancellation %s', async (when) => {
|
|
125
|
+
const context = fixture();
|
|
126
|
+
const reason = new Error('replication aborted');
|
|
127
|
+
if (when == 'before count') {
|
|
128
|
+
context.controller.abort(reason);
|
|
129
|
+
} else if (when == 'during count') {
|
|
130
|
+
context.count.mockImplementation(async () => {
|
|
131
|
+
context.controller.abort(reason);
|
|
132
|
+
throw reason;
|
|
133
|
+
});
|
|
134
|
+
} else {
|
|
135
|
+
context.count.mockRejectedValue(new Error('count failed'));
|
|
136
|
+
context.next.mockImplementation(async () => {
|
|
137
|
+
context.controller.abort(reason);
|
|
138
|
+
throw reason;
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
await expect(context.estimate()).rejects.toBe(reason);
|
|
142
|
+
if (when != 'during sample') {
|
|
143
|
+
expect(context.metadata).not.toHaveBeenCalled();
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { estimateFilteredSnapshotCount } from '@module/replication/estimateFilteredSnapshotCount.js';
|
|
2
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
|
+
import { createLogger } from '@powersync/lib-services-framework';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { describe, expect, test, vi } from 'vitest';
|
|
6
|
+
import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
|
|
7
|
+
import { connectMongoData } from './util.js';
|
|
8
|
+
|
|
9
|
+
const LOGGER = createLogger({ silent: true });
|
|
10
|
+
|
|
11
|
+
describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('filtered snapshot count integration', () => {
|
|
12
|
+
test('counts with simple collation even when the collection uses case-insensitive collation', async () => {
|
|
13
|
+
const { client, db } = await connectMongoData();
|
|
14
|
+
const collection = await db.createCollection(`snapshot_count_${randomUUID()}`, {
|
|
15
|
+
collation: { locale: 'en', strength: 2 }
|
|
16
|
+
});
|
|
17
|
+
try {
|
|
18
|
+
await collection.insertMany([{ label: 'live' }, { label: 'LIVE' }, { label: 'other' }]);
|
|
19
|
+
expect(
|
|
20
|
+
await estimateFilteredSnapshotCount({
|
|
21
|
+
collection,
|
|
22
|
+
filter: { $expr: { $eq: ['$label', 'live'] } },
|
|
23
|
+
signal: new AbortController().signal,
|
|
24
|
+
logger: LOGGER
|
|
25
|
+
})
|
|
26
|
+
).toBe(1);
|
|
27
|
+
} finally {
|
|
28
|
+
await collection.drop();
|
|
29
|
+
await client.close();
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('executes the sample pipeline after a failed count and preserves binary filter semantics', async () => {
|
|
34
|
+
const { client, db } = await connectMongoData();
|
|
35
|
+
const collection = await db.createCollection(`snapshot_count_${randomUUID()}`, {
|
|
36
|
+
collation: { locale: 'en', strength: 2 }
|
|
37
|
+
});
|
|
38
|
+
try {
|
|
39
|
+
// For this small collection the sample contains one document. All rows have the same label,
|
|
40
|
+
// making the sampled result deterministic while exercising the real $sample/$facet pipeline.
|
|
41
|
+
await collection.insertMany(Array.from({ length: 50 }, () => ({ label: 'live' })));
|
|
42
|
+
const count = vi
|
|
43
|
+
.spyOn(collection, 'countDocuments')
|
|
44
|
+
.mockRejectedValue(new mongo.MongoOperationTimeoutError('timeout'));
|
|
45
|
+
try {
|
|
46
|
+
expect(
|
|
47
|
+
await estimateFilteredSnapshotCount({
|
|
48
|
+
collection,
|
|
49
|
+
filter: { $and: [{ label: 'live' }, { $expr: { $eq: ['$label', 'live'] } }] },
|
|
50
|
+
signal: new AbortController().signal,
|
|
51
|
+
logger: LOGGER
|
|
52
|
+
})
|
|
53
|
+
).toBe(50);
|
|
54
|
+
expect(
|
|
55
|
+
await estimateFilteredSnapshotCount({
|
|
56
|
+
collection,
|
|
57
|
+
filter: { $expr: { $eq: ['$label', 'LIVE'] } },
|
|
58
|
+
signal: new AbortController().signal,
|
|
59
|
+
logger: LOGGER
|
|
60
|
+
})
|
|
61
|
+
).toBe(-1);
|
|
62
|
+
} finally {
|
|
63
|
+
count.mockRestore();
|
|
64
|
+
}
|
|
65
|
+
} finally {
|
|
66
|
+
await collection.drop();
|
|
67
|
+
await client.close();
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
});
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
|
|
2
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
3
|
import { describe, expect, test, vi } from 'vitest';
|
|
3
4
|
import { openChangeStreamTestContext } from './change_stream_test_setup.js';
|
|
4
5
|
import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
|
|
@@ -6,7 +7,11 @@ import { describeWithStorage } from './util.js';
|
|
|
6
7
|
|
|
7
8
|
describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot query provider', () => {
|
|
8
9
|
describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
|
|
9
|
-
test.each([
|
|
10
|
+
test.each([
|
|
11
|
+
{ filtered: false, retryUnknown: false },
|
|
12
|
+
{ filtered: true, retryUnknown: false },
|
|
13
|
+
{ filtered: true, retryUnknown: true }
|
|
14
|
+
])('pages snapshots: %j', async ({ filtered, retryUnknown }) => {
|
|
10
15
|
// Run the same snapshot with and without a provider predicate. The filtered case selects even
|
|
11
16
|
// ids between 2 and 12 whose label is exactly 'live'; null must preserve the full snapshot.
|
|
12
17
|
const getSnapshotFilter = vi.fn(() =>
|
|
@@ -53,7 +58,21 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot quer
|
|
|
53
58
|
await collection.insertMany(
|
|
54
59
|
Array.from({ length: 14 }, (_, i) => ({ _id: i + 1, label: (i + 1) % 3 == 0 ? 'LIVE' : 'live' }))
|
|
55
60
|
);
|
|
56
|
-
|
|
61
|
+
const count = vi.spyOn(mongo.Collection.prototype, 'countDocuments');
|
|
62
|
+
const metadata = vi.spyOn(mongo.Collection.prototype, 'estimatedDocumentCount');
|
|
63
|
+
try {
|
|
64
|
+
if (retryUnknown) {
|
|
65
|
+
// Leave the queued estimate unknown, then let the snapshot retry against the real source.
|
|
66
|
+
count.mockRejectedValueOnce(new Error('count unavailable'));
|
|
67
|
+
metadata.mockRejectedValueOnce(new Error('metadata unavailable'));
|
|
68
|
+
}
|
|
69
|
+
await context.replicateSnapshot();
|
|
70
|
+
expect(count).toHaveBeenCalledTimes(retryUnknown ? 2 : filtered ? 1 : 0);
|
|
71
|
+
expect(metadata).toHaveBeenCalledTimes(retryUnknown ? 1 : filtered ? 0 : 2);
|
|
72
|
+
} finally {
|
|
73
|
+
count.mockRestore();
|
|
74
|
+
metadata.mockRestore();
|
|
75
|
+
}
|
|
57
76
|
const rows = (await context.getBucketData('global|0[]')).filter((op) => op.op == 'PUT');
|
|
58
77
|
// Matching ids 4, 8 and 10 require two pages at a page size of two. After the first page ends
|
|
59
78
|
// at id 8, pagination adds its own $expr requiring _id > 8. It must combine that condition
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
createMongoReplicationStreamBatch,
|
|
4
|
+
filterMongoReplicationStream
|
|
5
|
+
} from '@module/replication/MongoReplicationStream.js';
|
|
6
|
+
import type { MongoReplicationStreamBatch } from '@module/replication/MongoReplicationStreamTypes.js';
|
|
7
|
+
import { MongoReplicationFilterResult } from '@module/replication/MongoReplicationStreamTypes.js';
|
|
3
8
|
import { describe, expect, test, vi } from 'vitest';
|
|
4
9
|
import { openChangeStreamTestContext } from './change_stream_test_setup.js';
|
|
5
10
|
import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
|
|
@@ -41,31 +46,33 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
|
|
|
41
46
|
const provider = {
|
|
42
47
|
...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
|
|
43
48
|
openChangeStream: ({ open }: Parameters<typeof DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER.openChangeStream>[0]) =>
|
|
44
|
-
(async function* (): AsyncGenerator<
|
|
49
|
+
(async function* (): AsyncGenerator<MongoReplicationStreamBatch> {
|
|
45
50
|
let retained = false;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
if (
|
|
50
|
-
|
|
51
|
-
|
|
51
|
+
const filteredStream = filterMongoReplicationStream({
|
|
52
|
+
batches: open({}),
|
|
53
|
+
filter: (event) => {
|
|
54
|
+
if (event.ns.coll == 'documents') {
|
|
55
|
+
if (String(event.documentKey._id) != '1') {
|
|
56
|
+
return MongoReplicationFilterResult.EXCLUDE;
|
|
57
|
+
}
|
|
58
|
+
retained = true;
|
|
52
59
|
}
|
|
53
|
-
|
|
54
|
-
} else if (item.type == 'progress') {
|
|
55
|
-
// open({}) uses the shared reader, which emits progress with the batch's resume token
|
|
56
|
-
// after consuming a MongoDB batch, provided no split event is incomplete. This is a
|
|
57
|
-
// reader-generated boundary, not a MongoDB change event or a result of filtering.
|
|
58
|
-
// If a boundary arrives after retaining document 1 but before excluding any later
|
|
59
|
-
// documents, suppress it until a boundary also covers excluded changes. Forwarding
|
|
60
|
-
// that boundary triggers the injected flush failure with document 1 still pending,
|
|
61
|
-
// verifying that its write must be durable before the resume token can be saved.
|
|
62
|
-
if (retained && filtered == 0) continue;
|
|
63
|
-
yield { ...item, filteredCount: filtered };
|
|
64
|
-
retained = false;
|
|
65
|
-
filtered = 0;
|
|
66
|
-
continue;
|
|
60
|
+
return MongoReplicationFilterResult.KEEP;
|
|
67
61
|
}
|
|
68
|
-
|
|
62
|
+
});
|
|
63
|
+
for await (const batch of filteredStream) {
|
|
64
|
+
yield createMongoReplicationStreamBatch({
|
|
65
|
+
events: batch.events,
|
|
66
|
+
resolveResumeToken: () => {
|
|
67
|
+
const progress = batch.getResumeToken();
|
|
68
|
+
if (progress == null || (retained && batch.metadata.filteredCount == 0)) {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
retained = false;
|
|
72
|
+
return progress;
|
|
73
|
+
},
|
|
74
|
+
resolveMetadata: () => batch.metadata
|
|
75
|
+
});
|
|
69
76
|
}
|
|
70
77
|
})()
|
|
71
78
|
};
|
|
@@ -224,32 +231,43 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
|
|
|
224
231
|
createReplicationQueryProvider: () => ({
|
|
225
232
|
...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
|
|
226
233
|
openChangeStream({ open }) {
|
|
227
|
-
return (async function* (): AsyncGenerator<
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
234
|
+
return (async function* (): AsyncGenerator<MongoReplicationStreamBatch> {
|
|
235
|
+
for await (const batch of open({})) {
|
|
236
|
+
let count = 0;
|
|
237
|
+
const positionBeforeBatch = (await context.storage!.getStatus()).resumeLsn!;
|
|
238
|
+
const events = (function* () {
|
|
239
|
+
for (const change of batch.events) {
|
|
240
|
+
if ('ns' in change.event && change.event.ns.coll == 'documents') {
|
|
241
|
+
count++;
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
yield change;
|
|
245
|
+
}
|
|
246
|
+
})();
|
|
247
|
+
let forwardedProgress = false;
|
|
248
|
+
yield createMongoReplicationStreamBatch({
|
|
249
|
+
events: events,
|
|
250
|
+
resolveResumeToken: () => {
|
|
251
|
+
const progress = batch.getResumeToken();
|
|
252
|
+
if (progress == null || count == 0) {
|
|
253
|
+
return progress;
|
|
254
|
+
}
|
|
255
|
+
filtered += count;
|
|
256
|
+
beforeFailure = positionBeforeBatch;
|
|
257
|
+
if (scenario == 'adapter failure') throw new Error('injected adapter failure');
|
|
258
|
+
failFlush = scenario == 'flush failure';
|
|
259
|
+
forwardedProgress = true;
|
|
260
|
+
return progress;
|
|
261
|
+
},
|
|
262
|
+
resolveMetadata: () => ({ filteredCount: count })
|
|
263
|
+
});
|
|
264
|
+
if (forwardedProgress) {
|
|
265
|
+
// next() is requested only after this module has handled the batch, including its resume token. Inspect actual
|
|
246
266
|
// persisted state here, without sleeps or waiting for a later checkpoint event.
|
|
247
267
|
savedPositions.push((await context.storage!.getStatus()).resumeLsn!);
|
|
248
268
|
// End the successful case deliberately while the later checkpoint marker is
|
|
249
269
|
// still pending, so assertions observe progress independently of a commit.
|
|
250
270
|
if (savedPositions.length == 3) throw new Error('stopped after three safe boundaries');
|
|
251
|
-
} else {
|
|
252
|
-
yield item;
|
|
253
271
|
}
|
|
254
272
|
}
|
|
255
273
|
})();
|
package/test/tsconfig.json
CHANGED