@powersync/service-module-mongodb 0.0.0-dev-20261007110800 → 0.0.0-dev-20261015132441

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +14 -11
  2. package/benchmarks/env.ts +11 -0
  3. package/benchmarks/replication_branch_throughput.benchmark.ts +240 -0
  4. package/dist/api/MongoRouteAPIAdapter.d.ts +5 -0
  5. package/dist/api/MongoRouteAPIAdapter.js +11 -4
  6. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  7. package/dist/module/MongoModule.js +1 -0
  8. package/dist/module/MongoModule.js.map +1 -1
  9. package/dist/replication/ChangeStream.d.ts +2 -1
  10. package/dist/replication/ChangeStream.js +164 -150
  11. package/dist/replication/ChangeStream.js.map +1 -1
  12. package/dist/replication/ChangeStreamReplicationJob.js +17 -9
  13. package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
  14. package/dist/replication/MongoReplicationQueryProvider.d.ts +9 -11
  15. package/dist/replication/MongoReplicationQueryProvider.js.map +1 -1
  16. package/dist/replication/MongoReplicationStream.d.ts +37 -48
  17. package/dist/replication/MongoReplicationStream.js +157 -30
  18. package/dist/replication/MongoReplicationStream.js.map +1 -1
  19. package/dist/replication/MongoReplicationStreamTypes.d.ts +157 -0
  20. package/dist/replication/MongoReplicationStreamTypes.js +19 -0
  21. package/dist/replication/MongoReplicationStreamTypes.js.map +1 -0
  22. package/dist/replication/MongoSnapshotter.js +59 -30
  23. package/dist/replication/MongoSnapshotter.js.map +1 -1
  24. package/dist/replication/estimateFilteredSnapshotCount.d.ts +12 -0
  25. package/dist/replication/estimateFilteredSnapshotCount.js +79 -0
  26. package/dist/replication/estimateFilteredSnapshotCount.js.map +1 -0
  27. package/dist/replication/replication-index.d.ts +1 -0
  28. package/dist/replication/replication-index.js +1 -0
  29. package/dist/replication/replication-index.js.map +1 -1
  30. package/package.json +11 -10
  31. package/src/api/MongoRouteAPIAdapter.ts +12 -5
  32. package/src/module/MongoModule.ts +1 -0
  33. package/src/replication/ChangeStream.ts +183 -173
  34. package/src/replication/ChangeStreamReplicationJob.ts +16 -9
  35. package/src/replication/MongoReplicationQueryProvider.ts +9 -11
  36. package/src/replication/MongoReplicationStream.ts +196 -83
  37. package/src/replication/MongoReplicationStreamTypes.ts +169 -0
  38. package/src/replication/MongoSnapshotter.ts +64 -33
  39. package/src/replication/estimateFilteredSnapshotCount.ts +93 -0
  40. package/src/replication/replication-index.ts +1 -0
  41. package/test/src/change_event_memory.test.ts +33 -0
  42. package/test/src/change_stream_abort.test.ts +110 -0
  43. package/test/src/filtering_example.test.ts +65 -0
  44. package/test/src/filtering_lifecycle.test.ts +101 -0
  45. package/test/src/query_provider_availability.test.ts +1 -0
  46. package/test/src/query_provider_diagnostics.test.ts +55 -0
  47. package/test/src/replication_stream.test.ts +411 -34
  48. package/test/src/snapshot_count.test.ts +146 -0
  49. package/test/src/snapshot_count_integration.test.ts +70 -0
  50. package/test/src/snapshot_query_provider.test.ts +21 -2
  51. package/test/src/stream_progress.test.ts +62 -44
  52. package/test/tsconfig.json +2 -2
  53. package/tsconfig.tsbuildinfo +1 -1
  54. package/vitest.benchmark.config.ts +15 -0
@@ -0,0 +1,146 @@
1
+ import { estimateFilteredSnapshotCount } from '@module/replication/estimateFilteredSnapshotCount.js';
2
+ import { mongo } from '@powersync/lib-service-mongodb';
3
+ import { createLogger } from '@powersync/lib-services-framework';
4
+ import { afterEach, describe, expect, test, vi } from 'vitest';
5
+
6
+ const FILTER = { $expr: { $eq: ['$label', 'live'] } };
7
+ const LOGGER = createLogger({ silent: true });
8
+
9
+ function fixture() {
10
+ const client = new mongo.MongoClient('mongodb://localhost:27017');
11
+ const collection = client.db('snapshot_count_unit').collection('documents');
12
+ const controller = new AbortController();
13
+ const count = vi.spyOn(collection, 'countDocuments').mockResolvedValue(42);
14
+ const metadata = vi.spyOn(collection, 'estimatedDocumentCount').mockResolvedValue(100_000);
15
+ const cursor = collection.aggregate();
16
+ const next = vi.spyOn(cursor, 'next').mockResolvedValue({ sampled: [{ count: 1_000 }], matches: [{ count: 250 }] });
17
+ const close = vi.spyOn(cursor, 'close').mockResolvedValue();
18
+ const aggregate = vi.spyOn(collection, 'aggregate').mockReturnValue(cursor);
19
+ const estimate = () =>
20
+ estimateFilteredSnapshotCount({ collection, filter: FILTER, signal: controller.signal, logger: LOGGER });
21
+ return { controller, count, metadata, next, close, aggregate, estimate };
22
+ }
23
+
24
+ afterEach(() => {
25
+ vi.restoreAllMocks();
26
+ });
27
+
28
+ describe('filtered snapshot count', () => {
29
+ test('uses a bounded exact count with the snapshot collation', async () => {
30
+ const context = fixture();
31
+ expect(await context.estimate()).toBe(42);
32
+ expect(context.count).toHaveBeenCalledWith(FILTER, {
33
+ collation: { locale: 'simple' },
34
+ readConcern: 'majority',
35
+ timeoutMS: 10_000,
36
+ signal: context.controller.signal
37
+ });
38
+ expect(context.metadata).not.toHaveBeenCalled();
39
+ expect(context.aggregate).not.toHaveBeenCalled();
40
+ });
41
+
42
+ test('preserves an exact zero without sampling', async () => {
43
+ const context = fixture();
44
+ context.count.mockResolvedValue(0);
45
+ expect(await context.estimate()).toBe(0);
46
+ expect(context.metadata).not.toHaveBeenCalled();
47
+ });
48
+
49
+ test('samples before filtering and scales by collection metadata after a count timeout', async () => {
50
+ const context = fixture();
51
+ context.count.mockRejectedValue(new mongo.MongoOperationTimeoutError('count timed out'));
52
+ expect(await context.estimate()).toBe(25_000);
53
+ expect(context.aggregate).toHaveBeenCalledWith(
54
+ [
55
+ { $sample: { size: 1_000 } },
56
+ { $facet: { sampled: [{ $count: 'count' }], matches: [{ $match: FILTER }, { $count: 'count' }] } }
57
+ ],
58
+ expect.objectContaining({
59
+ collation: { locale: 'simple' },
60
+ readConcern: 'majority',
61
+ signal: context.controller.signal,
62
+ allowDiskUse: false
63
+ })
64
+ );
65
+ const timeoutMS = context.aggregate.mock.calls[0][1]!.timeoutMS!;
66
+ expect(timeoutMS).toBeGreaterThan(0);
67
+ expect(timeoutMS).toBeLessThanOrEqual(10_000);
68
+ expect(context.close).toHaveBeenCalled();
69
+ });
70
+
71
+ test.each([
72
+ [10, 1],
73
+ [100_000_000, 10_000]
74
+ ])('bounds the sample size for a collection of %s documents', async (total, sampleSize) => {
75
+ const context = fixture();
76
+ context.count.mockRejectedValue(new Error('count failed'));
77
+ context.metadata.mockResolvedValue(total);
78
+ await context.estimate();
79
+ expect(context.aggregate.mock.calls[0][0]![0]).toEqual({ $sample: { size: sampleSize } });
80
+ });
81
+
82
+ test('uses the actual number sampled when metadata or collection size has changed', async () => {
83
+ const context = fixture();
84
+ context.count.mockRejectedValue(new Error('count failed'));
85
+ context.next.mockResolvedValue({ sampled: [{ count: 80 }], matches: [{ count: 20 }] });
86
+ expect(await context.estimate()).toBe(25_000);
87
+ });
88
+
89
+ test.each([null, { sampled: [], matches: [] }, { sampled: [{ count: 1_000 }], matches: [] }])(
90
+ 'leaves the total unknown for an empty sample or no matches: %j',
91
+ async (sample) => {
92
+ const context = fixture();
93
+ context.count.mockRejectedValue(new Error('count failed'));
94
+ context.next.mockResolvedValue(sample);
95
+ expect(await context.estimate()).toBe(-1);
96
+ expect(context.close).toHaveBeenCalled();
97
+ }
98
+ );
99
+
100
+ test.each(['metadata', 'sample', 'empty metadata', 'budget exhausted'])(
101
+ 'leaves the total unknown after %s',
102
+ async (failure) => {
103
+ const context = fixture();
104
+ context.count.mockRejectedValue(new Error('count failed'));
105
+ if (failure == 'metadata') {
106
+ context.metadata.mockRejectedValue(new Error('metadata failed'));
107
+ } else if (failure == 'sample') {
108
+ context.next.mockRejectedValue(new mongo.MongoOperationTimeoutError('sample timed out'));
109
+ } else if (failure == 'empty metadata') {
110
+ context.metadata.mockResolvedValue(0);
111
+ } else {
112
+ const { performance } = await import('node:perf_hooks');
113
+ vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValue(10_001);
114
+ }
115
+ expect(await context.estimate()).toBe(-1);
116
+ if (failure == 'sample') {
117
+ expect(context.close).toHaveBeenCalled();
118
+ } else {
119
+ expect(context.aggregate).not.toHaveBeenCalled();
120
+ }
121
+ }
122
+ );
123
+
124
+ test.each(['before count', 'during count', 'during sample'])('propagates cancellation %s', async (when) => {
125
+ const context = fixture();
126
+ const reason = new Error('replication aborted');
127
+ if (when == 'before count') {
128
+ context.controller.abort(reason);
129
+ } else if (when == 'during count') {
130
+ context.count.mockImplementation(async () => {
131
+ context.controller.abort(reason);
132
+ throw reason;
133
+ });
134
+ } else {
135
+ context.count.mockRejectedValue(new Error('count failed'));
136
+ context.next.mockImplementation(async () => {
137
+ context.controller.abort(reason);
138
+ throw reason;
139
+ });
140
+ }
141
+ await expect(context.estimate()).rejects.toBe(reason);
142
+ if (when != 'during sample') {
143
+ expect(context.metadata).not.toHaveBeenCalled();
144
+ }
145
+ });
146
+ });
@@ -0,0 +1,70 @@
1
+ import { estimateFilteredSnapshotCount } from '@module/replication/estimateFilteredSnapshotCount.js';
2
+ import { mongo } from '@powersync/lib-service-mongodb';
3
+ import { createLogger } from '@powersync/lib-services-framework';
4
+ import { randomUUID } from 'node:crypto';
5
+ import { describe, expect, test, vi } from 'vitest';
6
+ import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
7
+ import { connectMongoData } from './util.js';
8
+
9
+ const LOGGER = createLogger({ silent: true });
10
+
11
+ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('filtered snapshot count integration', () => {
12
+ test('counts with simple collation even when the collection uses case-insensitive collation', async () => {
13
+ const { client, db } = await connectMongoData();
14
+ const collection = await db.createCollection(`snapshot_count_${randomUUID()}`, {
15
+ collation: { locale: 'en', strength: 2 }
16
+ });
17
+ try {
18
+ await collection.insertMany([{ label: 'live' }, { label: 'LIVE' }, { label: 'other' }]);
19
+ expect(
20
+ await estimateFilteredSnapshotCount({
21
+ collection,
22
+ filter: { $expr: { $eq: ['$label', 'live'] } },
23
+ signal: new AbortController().signal,
24
+ logger: LOGGER
25
+ })
26
+ ).toBe(1);
27
+ } finally {
28
+ await collection.drop();
29
+ await client.close();
30
+ }
31
+ });
32
+
33
+ test('executes the sample pipeline after a failed count and preserves binary filter semantics', async () => {
34
+ const { client, db } = await connectMongoData();
35
+ const collection = await db.createCollection(`snapshot_count_${randomUUID()}`, {
36
+ collation: { locale: 'en', strength: 2 }
37
+ });
38
+ try {
39
+ // For this small collection the sample contains one document. All rows have the same label,
40
+ // making the sampled result deterministic while exercising the real $sample/$facet pipeline.
41
+ await collection.insertMany(Array.from({ length: 50 }, () => ({ label: 'live' })));
42
+ const count = vi
43
+ .spyOn(collection, 'countDocuments')
44
+ .mockRejectedValue(new mongo.MongoOperationTimeoutError('timeout'));
45
+ try {
46
+ expect(
47
+ await estimateFilteredSnapshotCount({
48
+ collection,
49
+ filter: { $and: [{ label: 'live' }, { $expr: { $eq: ['$label', 'live'] } }] },
50
+ signal: new AbortController().signal,
51
+ logger: LOGGER
52
+ })
53
+ ).toBe(50);
54
+ expect(
55
+ await estimateFilteredSnapshotCount({
56
+ collection,
57
+ filter: { $expr: { $eq: ['$label', 'LIVE'] } },
58
+ signal: new AbortController().signal,
59
+ logger: LOGGER
60
+ })
61
+ ).toBe(-1);
62
+ } finally {
63
+ count.mockRestore();
64
+ }
65
+ } finally {
66
+ await collection.drop();
67
+ await client.close();
68
+ }
69
+ });
70
+ });
@@ -1,4 +1,5 @@
1
1
  import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
2
+ import { mongo } from '@powersync/lib-service-mongodb';
2
3
  import { describe, expect, test, vi } from 'vitest';
3
4
  import { openChangeStreamTestContext } from './change_stream_test_setup.js';
4
5
  import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
@@ -6,7 +7,11 @@ import { describeWithStorage } from './util.js';
6
7
 
7
8
  describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot query provider', () => {
8
9
  describeWithStorage({ timeout: 30_000 }, ({ factory, storageVersion }) => {
9
- test.each([false, true])('pages snapshots with an expression filter: %s', async (filtered) => {
10
+ test.each([
11
+ { filtered: false, retryUnknown: false },
12
+ { filtered: true, retryUnknown: false },
13
+ { filtered: true, retryUnknown: true }
14
+ ])('pages snapshots: %j', async ({ filtered, retryUnknown }) => {
10
15
  // Run the same snapshot with and without a provider predicate. The filtered case selects even
11
16
  // ids between 2 and 12 whose label is exactly 'live'; null must preserve the full snapshot.
12
17
  const getSnapshotFilter = vi.fn(() =>
@@ -53,7 +58,21 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB snapshot quer
53
58
  await collection.insertMany(
54
59
  Array.from({ length: 14 }, (_, i) => ({ _id: i + 1, label: (i + 1) % 3 == 0 ? 'LIVE' : 'live' }))
55
60
  );
56
- await context.replicateSnapshot();
61
+ const count = vi.spyOn(mongo.Collection.prototype, 'countDocuments');
62
+ const metadata = vi.spyOn(mongo.Collection.prototype, 'estimatedDocumentCount');
63
+ try {
64
+ if (retryUnknown) {
65
+ // Leave the queued estimate unknown, then let the snapshot retry against the real source.
66
+ count.mockRejectedValueOnce(new Error('count unavailable'));
67
+ metadata.mockRejectedValueOnce(new Error('metadata unavailable'));
68
+ }
69
+ await context.replicateSnapshot();
70
+ expect(count).toHaveBeenCalledTimes(retryUnknown ? 2 : filtered ? 1 : 0);
71
+ expect(metadata).toHaveBeenCalledTimes(retryUnknown ? 1 : filtered ? 0 : 2);
72
+ } finally {
73
+ count.mockRestore();
74
+ metadata.mockRestore();
75
+ }
57
76
  const rows = (await context.getBucketData('global|0[]')).filter((op) => op.op == 'PUT');
58
77
  // Matching ids 4, 8 and 10 require two pages at a page size of two. After the first page ends
59
78
  // at id 8, pagination adds its own $expr requiring _id > 8. It must combine that condition
@@ -1,5 +1,10 @@
1
1
  import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
2
- import { MongoReplicationStreamItem } from '@module/replication/MongoReplicationStream.js';
2
+ import {
3
+ createMongoReplicationStreamBatch,
4
+ filterMongoReplicationStream
5
+ } from '@module/replication/MongoReplicationStream.js';
6
+ import type { MongoReplicationStreamBatch } from '@module/replication/MongoReplicationStreamTypes.js';
7
+ import { MongoReplicationFilterResult } from '@module/replication/MongoReplicationStreamTypes.js';
3
8
  import { describe, expect, test, vi } from 'vitest';
4
9
  import { openChangeStreamTestContext } from './change_stream_test_setup.js';
5
10
  import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
@@ -41,31 +46,33 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
41
46
  const provider = {
42
47
  ...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
43
48
  openChangeStream: ({ open }: Parameters<typeof DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER.openChangeStream>[0]) =>
44
- (async function* (): AsyncGenerator<MongoReplicationStreamItem> {
49
+ (async function* (): AsyncGenerator<MongoReplicationStreamBatch> {
45
50
  let retained = false;
46
- let filtered = 0;
47
- for await (const item of open({})) {
48
- if (item.type == 'change' && 'documentKey' in item.event && item.event.ns.coll == 'documents') {
49
- if (String(item.event.documentKey._id) != '1') {
50
- filtered++;
51
- continue;
51
+ const filteredStream = filterMongoReplicationStream({
52
+ batches: open({}),
53
+ filter: (event) => {
54
+ if (event.ns.coll == 'documents') {
55
+ if (String(event.documentKey._id) != '1') {
56
+ return MongoReplicationFilterResult.EXCLUDE;
57
+ }
58
+ retained = true;
52
59
  }
53
- retained = true;
54
- } else if (item.type == 'progress') {
55
- // open({}) uses the shared reader, which emits progress with the batch's resume token
56
- // after consuming a MongoDB batch, provided no split event is incomplete. This is a
57
- // reader-generated boundary, not a MongoDB change event or a result of filtering.
58
- // If a boundary arrives after retaining document 1 but before excluding any later
59
- // documents, suppress it until a boundary also covers excluded changes. Forwarding
60
- // that boundary triggers the injected flush failure with document 1 still pending,
61
- // verifying that its write must be durable before the resume token can be saved.
62
- if (retained && filtered == 0) continue;
63
- yield { ...item, filteredCount: filtered };
64
- retained = false;
65
- filtered = 0;
66
- continue;
60
+ return MongoReplicationFilterResult.KEEP;
67
61
  }
68
- yield item;
62
+ });
63
+ for await (const batch of filteredStream) {
64
+ yield createMongoReplicationStreamBatch({
65
+ events: batch.events,
66
+ resolveResumeToken: () => {
67
+ const progress = batch.getResumeToken();
68
+ if (progress == null || (retained && batch.metadata.filteredCount == 0)) {
69
+ return undefined;
70
+ }
71
+ retained = false;
72
+ return progress;
73
+ },
74
+ resolveMetadata: () => batch.metadata
75
+ });
69
76
  }
70
77
  })()
71
78
  };
@@ -224,32 +231,43 @@ describe.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('MongoDB durable adapt
224
231
  createReplicationQueryProvider: () => ({
225
232
  ...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
226
233
  openChangeStream({ open }) {
227
- return (async function* (): AsyncGenerator<MongoReplicationStreamItem> {
228
- let count = 0;
229
- for await (const item of open({})) {
230
- // Drop data events but keep the shared reader's batch boundaries and any control
231
- // events. Report how many complete changes each forwarded boundary excludes.
232
- if (item.type == 'change' && 'ns' in item.event && item.event.ns.coll == 'documents') {
233
- count++;
234
- continue;
235
- }
236
- if (item.type == 'progress' && count > 0) {
237
- filtered += count;
238
- beforeFailure = (await context.storage!.getStatus()).resumeLsn!;
239
- // The adapter error prevents delivery of the boundary. The flush error allows
240
- // delivery but fails storage processing before its resume token can be saved.
241
- if (scenario == 'adapter failure') throw new Error('injected adapter failure');
242
- failFlush = scenario == 'flush failure';
243
- yield { ...item, filteredCount: count };
244
- count = 0;
245
- // next() is requested only after this module has handled the progress item. Inspect actual
234
+ return (async function* (): AsyncGenerator<MongoReplicationStreamBatch> {
235
+ for await (const batch of open({})) {
236
+ let count = 0;
237
+ const positionBeforeBatch = (await context.storage!.getStatus()).resumeLsn!;
238
+ const events = (function* () {
239
+ for (const change of batch.events) {
240
+ if ('ns' in change.event && change.event.ns.coll == 'documents') {
241
+ count++;
242
+ continue;
243
+ }
244
+ yield change;
245
+ }
246
+ })();
247
+ let forwardedProgress = false;
248
+ yield createMongoReplicationStreamBatch({
249
+ events: events,
250
+ resolveResumeToken: () => {
251
+ const progress = batch.getResumeToken();
252
+ if (progress == null || count == 0) {
253
+ return progress;
254
+ }
255
+ filtered += count;
256
+ beforeFailure = positionBeforeBatch;
257
+ if (scenario == 'adapter failure') throw new Error('injected adapter failure');
258
+ failFlush = scenario == 'flush failure';
259
+ forwardedProgress = true;
260
+ return progress;
261
+ },
262
+ resolveMetadata: () => ({ filteredCount: count })
263
+ });
264
+ if (forwardedProgress) {
265
+ // next() is requested only after this module has handled the batch, including its resume token. Inspect actual
246
266
  // persisted state here, without sleeps or waiting for a later checkpoint event.
247
267
  savedPositions.push((await context.storage!.getStatus()).resumeLsn!);
248
268
  // End the successful case deliberately while the later checkpoint marker is
249
269
  // still pending, so assertions observe progress independently of a commit.
250
270
  if (savedPositions.length == 3) throw new Error('stopped after three safe boundaries');
251
- } else {
252
- yield item;
253
271
  }
254
272
  }
255
273
  })();
@@ -6,9 +6,9 @@
6
6
  "@module/*": ["../src/*"],
7
7
  "@core-tests/*": ["../../../packages/service-core/test/src/*"]
8
8
  },
9
- "rootDir": "src"
9
+ "rootDir": ".."
10
10
  },
11
- "include": ["src"],
11
+ "include": ["src", "../benchmarks"],
12
12
  "references": [
13
13
  {
14
14
  "path": "../"