@powersync/service-module-mongodb 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +153 -5
- package/dist/api/MongoRouteAPIAdapter.d.ts +4 -2
- package/dist/api/MongoRouteAPIAdapter.js +23 -32
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/common/SentinelLSN.d.ts +37 -0
- package/dist/common/SentinelLSN.js +59 -0
- package/dist/common/SentinelLSN.js.map +1 -0
- package/dist/module/MongoModule.js +2 -1
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +24 -0
- package/dist/replication/ChangeStream.js +203 -67
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +3 -2
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/ChangeStreamReplicator.js +1 -1
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/dist/replication/MongoRelation.d.ts +36 -1
- package/dist/replication/MongoRelation.js +133 -18
- package/dist/replication/MongoRelation.js.map +1 -1
- package/dist/replication/MongoSnapshotter.d.ts +9 -1
- package/dist/replication/MongoSnapshotter.js +129 -67
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +4 -1
- package/dist/replication/RawChangeStream.js +6 -10
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/checkpoints/CheckpointImplementation.d.ts +135 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +58 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +224 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +27 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +116 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
- package/dist/replication/replication-utils.d.ts +6 -0
- package/dist/replication/replication-utils.js +19 -2
- package/dist/replication/replication-utils.js.map +1 -1
- package/dist/types/types.d.ts +7 -2
- package/dist/types/types.js +20 -2
- package/dist/types/types.js.map +1 -1
- package/package.json +11 -21
- package/src/api/MongoRouteAPIAdapter.ts +28 -39
- package/src/common/SentinelLSN.ts +78 -0
- package/src/module/MongoModule.ts +2 -1
- package/src/replication/ChangeStream.ts +223 -78
- package/src/replication/ChangeStreamReplicationJob.ts +3 -2
- package/src/replication/ChangeStreamReplicator.ts +1 -1
- package/src/replication/MongoRelation.ts +153 -21
- package/src/replication/MongoSnapshotter.ts +148 -73
- package/src/replication/RawChangeStream.ts +27 -32
- package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
- package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +269 -0
- package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +145 -0
- package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
- package/src/replication/replication-utils.ts +23 -2
- package/src/types/types.ts +27 -2
- package/test/DOCUMENTDB_TESTING.md +115 -0
- package/test/src/DatabaseType.ts +25 -0
- package/test/src/change_stream.test.ts +180 -71
- package/test/src/change_stream_utils.ts +117 -29
- package/test/src/checkpoint_retry.test.ts +134 -0
- package/test/src/config.test.ts +34 -0
- package/test/src/documentdb_helpers.test.ts +124 -0
- package/test/src/documentdb_mode.test.ts +1039 -0
- package/test/src/mongo_test.test.ts +15 -5
- package/test/src/raw_change_stream.test.ts +147 -126
- package/test/src/resume.test.ts +2 -2
- package/test/src/resume_token.test.ts +30 -0
- package/test/src/slow_tests.test.ts +4 -1
- package/test/src/test-timeouts.ts +23 -0
- package/test/src/util.ts +14 -32
- package/tsconfig.json +1 -4
- package/tsconfig.scripts.json +1 -2
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -13,9 +13,10 @@ import {
|
|
|
13
13
|
TimeValuePrecision
|
|
14
14
|
} from '@powersync/service-sync-rules';
|
|
15
15
|
|
|
16
|
-
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
16
|
+
import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
17
|
+
import { ObjectId } from 'bson';
|
|
17
18
|
import { MongoLSN } from '../common/MongoLSN.js';
|
|
18
|
-
import {
|
|
19
|
+
import { SentinelLSN } from '../common/SentinelLSN.js';
|
|
19
20
|
|
|
20
21
|
export function getMongoRelation(
|
|
21
22
|
source: mongo.ChangeStreamNameSpace,
|
|
@@ -174,32 +175,163 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
|
|
|
174
175
|
*/
|
|
175
176
|
export const STANDALONE_CHECKPOINT_ID = '_standalone_checkpoint';
|
|
176
177
|
|
|
177
|
-
|
|
178
|
+
/**
|
|
179
|
+
* Id for checkpoint records managed by the {@link SentinelCheckpointImplementation} implementation.
|
|
180
|
+
*/
|
|
181
|
+
export const SENTINEL_CHECKPOINT_ID = '_sentinel_checkpoint';
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Create a checkpoint by upserting a document in _powersync_checkpoints, and
|
|
185
|
+
* return a comparable LSN string derived from the write's operationTime.
|
|
186
|
+
*
|
|
187
|
+
* Standard MongoDB only: this requires session.operationTime, which DocumentDB
|
|
188
|
+
* does not provide (it throws PSYNC_S1004 when it is missing). The DocumentDB /
|
|
189
|
+
* sentinel path builds LSNs via {@link createSentinelCheckpointLsn} instead.
|
|
190
|
+
*/
|
|
191
|
+
export async function createCheckpoint(db: mongo.Db, id: mongo.ObjectId | string): Promise<string> {
|
|
192
|
+
const TRIES = 2;
|
|
193
|
+
for (let i = 0; i < TRIES; i++) {
|
|
194
|
+
try {
|
|
195
|
+
return await createCheckpointInner(db, id);
|
|
196
|
+
} catch (e) {
|
|
197
|
+
if (i < TRIES - 1) {
|
|
198
|
+
logger.warn(`Failed to create checkpoint on attempt ${i + 1}`, e);
|
|
199
|
+
} else {
|
|
200
|
+
throw e;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
throw new ServiceAssertionError(`Unreachable code`);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async function createCheckpointInner(db: mongo.Db, id: mongo.ObjectId | string): Promise<string> {
|
|
208
|
+
// We use an unique id per process, and clear documents on startup.
|
|
209
|
+
// This is so that we can filter events for our own process only, and ignore
|
|
210
|
+
// events from other processes.
|
|
211
|
+
|
|
212
|
+
// We use a command instead of a regular update to avoid auto retries on writes.
|
|
213
|
+
// An auto retry on the write can trigger a weird edge case where the change stream event
|
|
214
|
+
// has the clusterTime of the first write, while the returned operation time is for the second no-op write.
|
|
215
|
+
// Instead, we do manual retries, which does not have the same write de-duplication logic.
|
|
216
|
+
// A sentinal-based approach would be better here, but that is a much bigger change.
|
|
217
|
+
|
|
218
|
+
const update: mongo.Document = { $inc: { i: 1 } };
|
|
219
|
+
|
|
220
|
+
const response = await db.command({
|
|
221
|
+
findAndModify: '_powersync_checkpoints',
|
|
222
|
+
query: {
|
|
223
|
+
_id: id as any
|
|
224
|
+
},
|
|
225
|
+
new: true,
|
|
226
|
+
upsert: true,
|
|
227
|
+
update
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const time = response.operationTime as mongo.Timestamp | undefined;
|
|
231
|
+
if (time == null) {
|
|
232
|
+
throw new ServiceError(ErrorCode.PSYNC_S1004, `clusterTime not available for checkpoint`);
|
|
233
|
+
}
|
|
234
|
+
return new MongoLSN({ timestamp: time }).comparable;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Create a DocumentDB comparable LSN by advancing the shared sentinel checkpoint
|
|
239
|
+
* document ({@link SENTINEL_CHECKPOINT_ID}). The returned LSN encodes the
|
|
240
|
+
* checkpoint counter in the same 16-hex shape as a MongoDB timestamp LSN (see
|
|
241
|
+
* {@link SentinelLSN}), so the two formats are directly comparable. The counter is
|
|
242
|
+
* seeded in the epoch-seconds range (see below), so a sentinel LSN always sorts
|
|
243
|
+
* above any real-timestamp LSN issued in the past.
|
|
244
|
+
*
|
|
245
|
+
* This counter is intentionally global to the source database. It is used for
|
|
246
|
+
* storage/client checkpoint comparisons and write checkpoint heads, so it must
|
|
247
|
+
* not reset when a new ChangeStream instance or new sync rules start.
|
|
248
|
+
*
|
|
249
|
+
* The document is a single shared record whose `stream_id` field alternates:
|
|
250
|
+
* batch/keepalive bumps stamp it with the calling stream's id (so a stream can
|
|
251
|
+
* recognise its own private barriers), while standalone bumps (write checkpoint
|
|
252
|
+
* heads, snapshot markers) clear it to null. `i` advances globally regardless.
|
|
253
|
+
*
|
|
254
|
+
* @param changeStreamId
|
|
255
|
+
* When provided, the bump is attributed to this stream (a private barrier).
|
|
256
|
+
* When omitted, it is a standalone bump and stream_id is cleared to null.
|
|
257
|
+
*/
|
|
258
|
+
export async function createSentinelCheckpointLsn(
|
|
178
259
|
client: mongo.MongoClient,
|
|
179
260
|
db: mongo.Db,
|
|
180
|
-
|
|
261
|
+
changeStreamId?: ObjectId
|
|
181
262
|
): Promise<string> {
|
|
182
263
|
const session = client.startSession();
|
|
183
264
|
try {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
265
|
+
const collection = db.collection('_powersync_checkpoints');
|
|
266
|
+
|
|
267
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
268
|
+
// Common path: increment the existing counter.
|
|
269
|
+
const result = await collection.findOneAndUpdate(
|
|
270
|
+
{
|
|
271
|
+
_id: SENTINEL_CHECKPOINT_ID as any,
|
|
272
|
+
i: { $exists: true }
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
$inc: { i: 1 },
|
|
276
|
+
// Standalone bumps (no changeStreamId) must explicitly clear the
|
|
277
|
+
// stream_id left by a previous batch bump — this is a single shared
|
|
278
|
+
// document whose stream_id alternates. Write null rather than relying
|
|
279
|
+
// on `undefined` being serialized (which depends on the driver's
|
|
280
|
+
// ignoreUndefined option, and collapses to an empty $set if enabled).
|
|
281
|
+
$set: {
|
|
282
|
+
stream_id: changeStreamId ?? null
|
|
283
|
+
}
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
returnDocument: 'after',
|
|
287
|
+
session
|
|
288
|
+
}
|
|
289
|
+
);
|
|
290
|
+
|
|
291
|
+
if (result != null) {
|
|
292
|
+
// `i` is a bigint: the client is configured with useBigInt64, and the
|
|
293
|
+
// seed exceeds 2^53 so it could not be safely represented as a number.
|
|
294
|
+
return new SentinelLSN({ sentinel: result.i }).comparable;
|
|
198
295
|
}
|
|
296
|
+
|
|
297
|
+
// The counter document does not exist: first run, or a consumer deleted
|
|
298
|
+
// it in their source database. Seed the counter in the epoch-SECONDS range
|
|
299
|
+
// (seconds in the high 32 bits, mirroring a MongoDB timestamp) rather than
|
|
300
|
+
// starting at 1. Two properties follow:
|
|
301
|
+
//
|
|
302
|
+
// - A sentinel LSN sorts above any real-timestamp LSN issued in the past,
|
|
303
|
+
// because its high 32 bits are the current epoch seconds.
|
|
304
|
+
// - A re-created counter jumps forward instead of backward across deletion:
|
|
305
|
+
// the seed advances by 2^32 each wall-clock second, while checkpoints add
|
|
306
|
+
// 1 each, so any later re-seed exceeds the previously issued coordinate
|
|
307
|
+
// (keeping the LSN domain monotonic; otherwise new write checkpoint heads
|
|
308
|
+
// could resolve against old, higher committed LSNs).
|
|
309
|
+
//
|
|
310
|
+
// $setOnInsert cannot be combined with $inc on the same field, so this
|
|
311
|
+
// is a separate upsert; the loop then retries the increment. The
|
|
312
|
+
// $setOnInsert is a no-op if another process created the document
|
|
313
|
+
// concurrently.
|
|
314
|
+
await collection.updateOne(
|
|
315
|
+
{
|
|
316
|
+
_id: SENTINEL_CHECKPOINT_ID as any
|
|
317
|
+
},
|
|
318
|
+
{
|
|
319
|
+
$setOnInsert: {
|
|
320
|
+
i: BigInt(Math.floor(Date.now() / 1000)) << 32n,
|
|
321
|
+
stream_id: changeStreamId ?? null
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
upsert: true,
|
|
326
|
+
session
|
|
327
|
+
}
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
throw new ServiceError(
|
|
332
|
+
ErrorCode.PSYNC_S1301,
|
|
333
|
+
`Failed to increment the sentinel checkpoint counter - the checkpoint document may be getting deleted concurrently.`
|
|
199
334
|
);
|
|
200
|
-
const time = session.operationTime!;
|
|
201
|
-
// TODO: Use the above when we support custom write checkpoints
|
|
202
|
-
return new MongoLSN({ timestamp: time }).comparable;
|
|
203
335
|
} finally {
|
|
204
336
|
await session.endSession();
|
|
205
337
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { container, ErrorCode, Logger, ReplicationAbortedError, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import {
|
|
4
|
-
InternalOpId,
|
|
5
4
|
MetricsEngine,
|
|
6
5
|
PerformanceTracer,
|
|
7
6
|
SaveOperationTag,
|
|
@@ -15,11 +14,13 @@ import { performance } from 'node:perf_hooks';
|
|
|
15
14
|
import { MongoLSN } from '../common/MongoLSN.js';
|
|
16
15
|
import { PostImagesOption } from '../types/types.js';
|
|
17
16
|
import { escapeRegExp } from '../utils.js';
|
|
17
|
+
import { CheckpointImplementation } from './checkpoints/CheckpointImplementation.js';
|
|
18
|
+
import { createCheckpointImplementation } from './checkpoints/create-checkpoint-implementation.js';
|
|
18
19
|
import { MongoManager } from './MongoManager.js';
|
|
19
|
-
import {
|
|
20
|
+
import { getMongoRelation } from './MongoRelation.js';
|
|
20
21
|
import { ChunkedSnapshotQuery } from './MongoSnapshotQuery.js';
|
|
21
22
|
import { ChangeStreamBatch, parseChangeDocument, rawChangeStream } from './RawChangeStream.js';
|
|
22
|
-
import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
23
|
+
import { CHECKPOINTS_COLLECTION, detectDocumentDb } from './replication-utils.js';
|
|
23
24
|
import { DirectSourceRowConverter, SourceRowConverter } from './SourceRowConverter.js';
|
|
24
25
|
|
|
25
26
|
export interface MongoSnapshotterOptions {
|
|
@@ -71,9 +72,11 @@ export class MongoSnapshotter {
|
|
|
71
72
|
private readonly queue = new Set<SnapshotQueueItem>();
|
|
72
73
|
private initialSnapshotDone = Promise.withResolvers<void>();
|
|
73
74
|
private nextItemQueued: PromiseWithResolvers<void> | null = null;
|
|
74
|
-
private lastSnapshotOpId: InternalOpId | null = null;
|
|
75
75
|
private lastTouchedAt = performance.now();
|
|
76
76
|
|
|
77
|
+
private isDocumentDb = false;
|
|
78
|
+
private _checkpointImplementation: CheckpointImplementation | null = null;
|
|
79
|
+
|
|
77
80
|
constructor(options: MongoSnapshotterOptions) {
|
|
78
81
|
this.storage = options.storage;
|
|
79
82
|
this.metrics = options.metrics;
|
|
@@ -96,6 +99,9 @@ export class MongoSnapshotter {
|
|
|
96
99
|
this.abortSignal.addEventListener('abort', () => {
|
|
97
100
|
this.nextItemQueued?.resolve();
|
|
98
101
|
});
|
|
102
|
+
|
|
103
|
+
// Errors here should not result in uncaught rejection - calling waitForInitialSnapshot is optional.
|
|
104
|
+
void this.initialSnapshotDone.promise.catch(() => {});
|
|
99
105
|
}
|
|
100
106
|
|
|
101
107
|
private get usePostImages() {
|
|
@@ -110,23 +116,71 @@ export class MongoSnapshotter {
|
|
|
110
116
|
return this.storage.storageConfig.softDeleteCurrentData;
|
|
111
117
|
}
|
|
112
118
|
|
|
119
|
+
/** The active checkpoint strategy. Only valid after ensureDetected(). */
|
|
120
|
+
private get checkpointImplementation(): CheckpointImplementation {
|
|
121
|
+
if (this._checkpointImplementation == null) {
|
|
122
|
+
throw new ServiceError(
|
|
123
|
+
ErrorCode.PSYNC_S1301,
|
|
124
|
+
'Checkpoint implementation not initialized - call ensureDetected() first'
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
return this._checkpointImplementation;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Detect DocumentDB and select the checkpoint implementation. Idempotent.
|
|
132
|
+
* Also validates server topology (sharded/standalone) for standard MongoDB.
|
|
133
|
+
*/
|
|
134
|
+
private async ensureDetected(): Promise<void> {
|
|
135
|
+
if (this._checkpointImplementation != null) {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
this.isDocumentDb = await detectDocumentDb(this.defaultDb);
|
|
139
|
+
if (!this.isDocumentDb) {
|
|
140
|
+
const hello = await this.defaultDb.command({ hello: 1 });
|
|
141
|
+
if (hello.msg == 'isdbgrid') {
|
|
142
|
+
throw new ServiceError(
|
|
143
|
+
ErrorCode.PSYNC_S1341,
|
|
144
|
+
'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
|
|
145
|
+
);
|
|
146
|
+
} else if (hello.setName == null) {
|
|
147
|
+
throw new ServiceError(
|
|
148
|
+
ErrorCode.PSYNC_S1342,
|
|
149
|
+
'Standalone MongoDB instances are not supported - use a replicaset.'
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
|
|
154
|
+
client: this.client,
|
|
155
|
+
db: this.defaultDb,
|
|
156
|
+
checkpointStreamId: this.checkpointStreamId,
|
|
157
|
+
logger: this.logger
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
113
161
|
async checkSlot(): Promise<InitResult> {
|
|
114
162
|
const status = await this.storage.getStatus();
|
|
115
|
-
if (status.
|
|
163
|
+
if (status.snapshotDone) {
|
|
116
164
|
this.logger.info(`Initial replication already done`);
|
|
117
165
|
return { needsInitialSync: false, snapshotLsn: null };
|
|
118
166
|
}
|
|
119
167
|
|
|
120
|
-
return { needsInitialSync: true, snapshotLsn: status.
|
|
168
|
+
return { needsInitialSync: true, snapshotLsn: status.resumeLsn };
|
|
121
169
|
}
|
|
122
170
|
|
|
123
171
|
async setupCheckpointsCollection() {
|
|
172
|
+
await this.ensureDetected();
|
|
124
173
|
const collection = await this.getCollectionInfo(this.defaultDb.databaseName, CHECKPOINTS_COLLECTION);
|
|
125
174
|
if (collection == null) {
|
|
126
175
|
await this.defaultDb.createCollection(CHECKPOINTS_COLLECTION, {
|
|
127
|
-
|
|
176
|
+
// DocumentDB does not support changeStreamPreAndPostImages.
|
|
177
|
+
...(this.isDocumentDb ? {} : { changeStreamPreAndPostImages: { enabled: true } })
|
|
128
178
|
});
|
|
129
|
-
} else if (
|
|
179
|
+
} else if (
|
|
180
|
+
!this.isDocumentDb &&
|
|
181
|
+
this.usePostImages &&
|
|
182
|
+
collection.options?.changeStreamPreAndPostImages?.enabled != true
|
|
183
|
+
) {
|
|
130
184
|
// Drop + create requires less permissions than collMod,
|
|
131
185
|
// and we don't care about the data in this collection.
|
|
132
186
|
await this.defaultDb.dropCollection(CHECKPOINTS_COLLECTION);
|
|
@@ -137,17 +191,28 @@ export class MongoSnapshotter {
|
|
|
137
191
|
// Clear the collection on startup, to keep it clean
|
|
138
192
|
// We never query this collection directly, and don't want to keep the data around.
|
|
139
193
|
// We only use this to get data into the oplog/changestream.
|
|
140
|
-
|
|
194
|
+
//
|
|
195
|
+
// The implementation supplies the filter: the sentinel implementation must
|
|
196
|
+
// preserve the sentinel checkpoint document, since its counter is the
|
|
197
|
+
// globally-ordered component of every committed DocumentDB LSN.
|
|
198
|
+
await this.defaultDb
|
|
199
|
+
.collection(CHECKPOINTS_COLLECTION)
|
|
200
|
+
.deleteMany(this.checkpointImplementation.checkpointClearFilter);
|
|
141
201
|
}
|
|
142
202
|
}
|
|
143
203
|
|
|
144
204
|
async queueSnapshotTables(snapshotLsn: string | null) {
|
|
145
205
|
await this.client.connect();
|
|
206
|
+
// Ensure isDocumentDb is set before any getSourceNamespaceFilters() call below
|
|
207
|
+
// (notably the validateSnapshotLsn resume path, which does not go through
|
|
208
|
+
// getSnapshotLsn). Idempotent.
|
|
209
|
+
await this.ensureDetected();
|
|
146
210
|
await using writer = await this.storage.createWriter({
|
|
147
211
|
zeroLSN: MongoLSN.ZERO.comparable,
|
|
148
212
|
defaultSchema: this.defaultDb.databaseName,
|
|
149
213
|
storeCurrentData: false,
|
|
150
214
|
skipExistingRows: true,
|
|
215
|
+
signal: this.abortSignal,
|
|
151
216
|
tracer: new PerformanceTracer('MongoDB initial snapshot setup')
|
|
152
217
|
});
|
|
153
218
|
if (snapshotLsn == null) {
|
|
@@ -258,7 +323,7 @@ export class MongoSnapshotter {
|
|
|
258
323
|
for (const table of tables) {
|
|
259
324
|
await this.snapshotTable(batch, table);
|
|
260
325
|
}
|
|
261
|
-
const noCheckpointBefore = await
|
|
326
|
+
const noCheckpointBefore = await this.checkpointImplementation.createStandaloneCheckpoint();
|
|
262
327
|
|
|
263
328
|
await batch.markTableSnapshotDone(tables, noCheckpointBefore);
|
|
264
329
|
}
|
|
@@ -277,20 +342,16 @@ export class MongoSnapshotter {
|
|
|
277
342
|
}
|
|
278
343
|
|
|
279
344
|
const status = await this.storage.getStatus();
|
|
280
|
-
if (status.
|
|
345
|
+
if (status.snapshotDone) {
|
|
281
346
|
return;
|
|
282
347
|
}
|
|
283
348
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
maxOpId: lastOp,
|
|
291
|
-
signal: this.abortSignal
|
|
292
|
-
});
|
|
293
|
-
}
|
|
349
|
+
// Compact storage _after_ initial replication, but _before_ we switch to this replication stream.
|
|
350
|
+
// Keeping snapshot_done false until this completes makes this resumable after interruption.
|
|
351
|
+
// No checkpoint exists yet - storage defaults to its highest persisted op id.
|
|
352
|
+
await this.storage.compactInitialReplication({
|
|
353
|
+
signal: this.abortSignal
|
|
354
|
+
});
|
|
294
355
|
|
|
295
356
|
if (this.queue.size != 0) {
|
|
296
357
|
return;
|
|
@@ -301,12 +362,13 @@ export class MongoSnapshotter {
|
|
|
301
362
|
zeroLSN: MongoLSN.ZERO.comparable,
|
|
302
363
|
defaultSchema: this.defaultDb.databaseName,
|
|
303
364
|
storeCurrentData: false,
|
|
304
|
-
skipExistingRows: true
|
|
365
|
+
skipExistingRows: true,
|
|
366
|
+
signal: this.abortSignal
|
|
305
367
|
});
|
|
306
368
|
|
|
307
369
|
// The checkpoint here is a marker - we need to replicate up to at least this
|
|
308
370
|
// point before the data can be considered consistent.
|
|
309
|
-
const checkpoint = await
|
|
371
|
+
const checkpoint = await this.checkpointImplementation.createStandaloneCheckpoint();
|
|
310
372
|
if (this.queue.size != 0) {
|
|
311
373
|
return;
|
|
312
374
|
}
|
|
@@ -318,7 +380,7 @@ export class MongoSnapshotter {
|
|
|
318
380
|
// KLUDGE: We need to create an extra checkpoint _after_ marking the snapshot done, to fix
|
|
319
381
|
// issues with order of processing commits(). This is picked up by tests on postgres storage,
|
|
320
382
|
// the issue may be specific to that storage engine.
|
|
321
|
-
await
|
|
383
|
+
await this.checkpointImplementation.createStandaloneCheckpoint();
|
|
322
384
|
}
|
|
323
385
|
|
|
324
386
|
private async replicateTable(tableRequest: SourceTable) {
|
|
@@ -331,6 +393,7 @@ export class MongoSnapshotter {
|
|
|
331
393
|
storeCurrentData: false,
|
|
332
394
|
skipExistingRows: true,
|
|
333
395
|
hooks: this.storageHooks,
|
|
396
|
+
signal: this.abortSignal,
|
|
334
397
|
tracer: new PerformanceTracer('MongoDB snapshot table')
|
|
335
398
|
});
|
|
336
399
|
// Get fresh table info, in case it was updated while queuing.
|
|
@@ -342,16 +405,13 @@ export class MongoSnapshotter {
|
|
|
342
405
|
}
|
|
343
406
|
|
|
344
407
|
await this.snapshotTable(writer, table);
|
|
345
|
-
const noCheckpointBefore = await
|
|
408
|
+
const noCheckpointBefore = await this.checkpointImplementation.createStandaloneCheckpoint();
|
|
346
409
|
await writer.markTableSnapshotDone([table], noCheckpointBefore);
|
|
347
410
|
|
|
348
|
-
// This commit
|
|
411
|
+
// This commit durably records the persisted ops, so a later checkpoint covers them.
|
|
349
412
|
const resumeLsn = writer.resumeFromLsn ?? MongoLSN.ZERO.comparable;
|
|
350
413
|
await writer.commit(resumeLsn);
|
|
351
414
|
|
|
352
|
-
if (writer.last_flushed_op != null) {
|
|
353
|
-
this.lastSnapshotOpId = writer.last_flushed_op;
|
|
354
|
-
}
|
|
355
415
|
this.logger.info(`Flushed snapshot at ${writer.last_flushed_op}`);
|
|
356
416
|
}
|
|
357
417
|
|
|
@@ -364,6 +424,13 @@ export class MongoSnapshotter {
|
|
|
364
424
|
return [];
|
|
365
425
|
}
|
|
366
426
|
|
|
427
|
+
if (tablePattern.isSchemaWildcard) {
|
|
428
|
+
throw new ServiceError(
|
|
429
|
+
ErrorCode.PSYNC_R2201,
|
|
430
|
+
'Schema wildcards ("%") in table patterns are not supported for MongoDB connections.'
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
|
|
367
434
|
const nameFilter = tablePattern.isWildcard
|
|
368
435
|
? new RegExp('^' + escapeRegExp(tablePattern.tablePrefix))
|
|
369
436
|
: tablePattern.name;
|
|
@@ -445,10 +512,7 @@ export class MongoSnapshotter {
|
|
|
445
512
|
}
|
|
446
513
|
|
|
447
514
|
// Important: flush before marking progress
|
|
448
|
-
|
|
449
|
-
if (result?.flushed_op != null) {
|
|
450
|
-
this.lastSnapshotOpId = result.flushed_op;
|
|
451
|
-
}
|
|
515
|
+
await batch.flush();
|
|
452
516
|
at += docBatch.length;
|
|
453
517
|
rowsReplicatedMetric.add(docBatch.length);
|
|
454
518
|
|
|
@@ -481,10 +545,12 @@ export class MongoSnapshotter {
|
|
|
481
545
|
// Ignore the postImages check in this case.
|
|
482
546
|
}
|
|
483
547
|
|
|
548
|
+
// Note: resolveTables uses the batch's own parsed sync config set. Passing
|
|
549
|
+
// this.syncRules here would pair sources from one parse with the batch's mapping
|
|
550
|
+
// from another parse.
|
|
484
551
|
const result = await batch.resolveTables({
|
|
485
552
|
connection_id: this.connectionId,
|
|
486
|
-
source: descriptor
|
|
487
|
-
syncRules: this.syncRules
|
|
553
|
+
source: descriptor
|
|
488
554
|
});
|
|
489
555
|
|
|
490
556
|
// Drop conflicting collections.
|
|
@@ -526,19 +592,7 @@ export class MongoSnapshotter {
|
|
|
526
592
|
}
|
|
527
593
|
|
|
528
594
|
private async getSnapshotLsn(): Promise<string> {
|
|
529
|
-
|
|
530
|
-
// Basic sanity check
|
|
531
|
-
if (hello.msg == 'isdbgrid') {
|
|
532
|
-
throw new ServiceError(
|
|
533
|
-
ErrorCode.PSYNC_S1341,
|
|
534
|
-
'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
|
|
535
|
-
);
|
|
536
|
-
} else if (hello.setName == null) {
|
|
537
|
-
throw new ServiceError(
|
|
538
|
-
ErrorCode.PSYNC_S1342,
|
|
539
|
-
'Standalone MongoDB instances are not supported - use a replicaset.'
|
|
540
|
-
);
|
|
541
|
-
}
|
|
595
|
+
await this.ensureDetected();
|
|
542
596
|
|
|
543
597
|
// Open a change stream just to get a resume token for later use.
|
|
544
598
|
// We could use clusterTime from the hello command, but that won't tell us if the
|
|
@@ -551,14 +605,19 @@ export class MongoSnapshotter {
|
|
|
551
605
|
// Instead, we create a new checkpoint document, and wait until we get that document back in the stream.
|
|
552
606
|
// To avoid potential race conditions with the checkpoint creation, we create a new checkpoint document
|
|
553
607
|
// periodically until the timeout is reached.
|
|
608
|
+
//
|
|
609
|
+
// For the sentinel implementation (DocumentDB) there is no
|
|
610
|
+
// startAtOperationTime: the stream opens from "now" (lsn null) and the
|
|
611
|
+
// retry loop below re-creates checkpoints until one is observed.
|
|
554
612
|
|
|
555
613
|
const LSN_TIMEOUT_SECONDS = 60;
|
|
556
614
|
const LSN_CREATE_INTERVAL_SECONDS = 1;
|
|
557
615
|
|
|
558
|
-
|
|
616
|
+
this.checkpointImplementation.seedPosition(null);
|
|
617
|
+
const startStreamFromLsn = await this.checkpointImplementation.createFirstBarrier();
|
|
559
618
|
const filters = this.getSourceNamespaceFilters();
|
|
560
619
|
const iter = this.rawChangeStreamBatches({
|
|
561
|
-
lsn:
|
|
620
|
+
lsn: startStreamFromLsn,
|
|
562
621
|
maxAwaitTimeMS: 0,
|
|
563
622
|
signal: this.abortSignal,
|
|
564
623
|
filters
|
|
@@ -573,7 +632,7 @@ export class MongoSnapshotter {
|
|
|
573
632
|
break;
|
|
574
633
|
}
|
|
575
634
|
if (performance.now() - lastCheckpointCreated >= LSN_CREATE_INTERVAL_SECONDS * 1000) {
|
|
576
|
-
await
|
|
635
|
+
await this.checkpointImplementation.createBatchCheckpoint();
|
|
577
636
|
lastCheckpointCreated = performance.now();
|
|
578
637
|
}
|
|
579
638
|
batchesSeen += 1;
|
|
@@ -583,14 +642,15 @@ export class MongoSnapshotter {
|
|
|
583
642
|
const ns = 'ns' in changeDocument && 'coll' in changeDocument.ns ? changeDocument.ns : undefined;
|
|
584
643
|
|
|
585
644
|
if (ns?.coll == CHECKPOINTS_COLLECTION && 'documentKey' in changeDocument) {
|
|
586
|
-
const
|
|
587
|
-
if (
|
|
645
|
+
const kind = this.checkpointImplementation.event.observe(changeDocument);
|
|
646
|
+
if (kind != 'own-barrier') {
|
|
647
|
+
// Standalone events still feed the implementation's coordinate via
|
|
648
|
+
// event.observe above; we only resolve on our own barrier.
|
|
588
649
|
continue;
|
|
589
650
|
}
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
}).comparable;
|
|
651
|
+
// Observing our own barrier has set the coordinate (the barrier event
|
|
652
|
+
// carries the sentinel counter directly), so the LSN is ready.
|
|
653
|
+
return this.checkpointImplementation.event.lsn(changeDocument);
|
|
594
654
|
}
|
|
595
655
|
|
|
596
656
|
eventsSeen += 1;
|
|
@@ -648,7 +708,12 @@ export class MongoSnapshotter {
|
|
|
648
708
|
}
|
|
649
709
|
}
|
|
650
710
|
|
|
651
|
-
|
|
711
|
+
// DocumentDB always opens a cluster-level change stream, even in single-database
|
|
712
|
+
// mode, so a coll-only filter would match same-named collections in other
|
|
713
|
+
// databases of the cluster. Filter on the full namespace whenever the stream
|
|
714
|
+
// is cluster-scoped. See ChangeStream.getSourceNamespaceFilters for details.
|
|
715
|
+
const useFullNamespaceFilter = this.isDocumentDb || multipleDatabases;
|
|
716
|
+
const nsFilter = useFullNamespaceFilter
|
|
652
717
|
? { ns: { $in: inFilters } }
|
|
653
718
|
: { 'ns.coll': { $in: inFilters.map((ns) => ns.coll) } };
|
|
654
719
|
if (regexFilters.length > 0) {
|
|
@@ -665,12 +730,15 @@ export class MongoSnapshotter {
|
|
|
665
730
|
signal?: AbortSignal;
|
|
666
731
|
tracer?: PerformanceTracer<'changestream'>;
|
|
667
732
|
}): AsyncIterableIterator<ChangeStreamBatch> {
|
|
668
|
-
const
|
|
669
|
-
const startAfter =
|
|
670
|
-
const resumeAfter =
|
|
733
|
+
const position = options.lsn ? this.checkpointImplementation.parseResumePosition(options.lsn) : null;
|
|
734
|
+
const startAfter = position?.startAfter ?? undefined;
|
|
735
|
+
const resumeAfter = position?.resumeAfter ?? undefined;
|
|
671
736
|
|
|
672
737
|
let fullDocument: 'required' | 'updateLookup';
|
|
673
|
-
if (this.
|
|
738
|
+
if (this.isDocumentDb) {
|
|
739
|
+
// DocumentDB does not support changeStreamPreAndPostImages, so 'required' won't work.
|
|
740
|
+
fullDocument = 'updateLookup';
|
|
741
|
+
} else if (this.usePostImages) {
|
|
674
742
|
// 'read_only' or 'auto_configure'
|
|
675
743
|
// Configuration happens during snapshot, or when we see new
|
|
676
744
|
// collections.
|
|
@@ -679,27 +747,32 @@ export class MongoSnapshotter {
|
|
|
679
747
|
fullDocument = 'updateLookup';
|
|
680
748
|
}
|
|
681
749
|
const streamOptions: mongo.ChangeStreamOptions & mongo.Document = {
|
|
682
|
-
showExpandedEvents: true,
|
|
683
750
|
fullDocument
|
|
684
751
|
};
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
];
|
|
752
|
+
if (!this.isDocumentDb) {
|
|
753
|
+
// DocumentDB does not support showExpandedEvents.
|
|
754
|
+
streamOptions.showExpandedEvents = true;
|
|
755
|
+
}
|
|
756
|
+
const pipeline: mongo.Document[] = [{ $changeStream: streamOptions }, { $match: options.filters.$match }];
|
|
757
|
+
if (!this.isDocumentDb) {
|
|
758
|
+
// DocumentDB does not support $changeStreamSplitLargeEvent.
|
|
759
|
+
pipeline.push({ $changeStreamSplitLargeEvent: {} });
|
|
760
|
+
}
|
|
690
761
|
|
|
691
762
|
// Only one of these options can be supplied at a time.
|
|
692
763
|
if (resumeAfter) {
|
|
693
764
|
streamOptions.resumeAfter = resumeAfter;
|
|
694
|
-
} else {
|
|
765
|
+
} else if (startAfter != null) {
|
|
695
766
|
// Legacy: We don't persist lsns without resumeTokens anymore, but we do still handle the
|
|
696
|
-
// case if we have an old one.
|
|
767
|
+
// case if we have an old one. The sentinel implementation never produces a startAfter,
|
|
768
|
+
// and a fresh DocumentDB stream opens from "now" with neither option set.
|
|
697
769
|
streamOptions.startAtOperationTime = startAfter;
|
|
698
770
|
}
|
|
699
771
|
|
|
700
772
|
let watchDb: mongo.Db;
|
|
701
|
-
if (options.filters.multipleDatabases) {
|
|
702
|
-
//
|
|
773
|
+
if (this.isDocumentDb || options.filters.multipleDatabases) {
|
|
774
|
+
// DocumentDB only supports cluster-level change streams.
|
|
775
|
+
// Requires readAnyDatabase@admin on Atlas.
|
|
703
776
|
watchDb = this.client.db('admin');
|
|
704
777
|
streamOptions.allChangesForCluster = true;
|
|
705
778
|
} else {
|
|
@@ -707,9 +780,11 @@ export class MongoSnapshotter {
|
|
|
707
780
|
watchDb = this.defaultDb;
|
|
708
781
|
}
|
|
709
782
|
|
|
783
|
+
const maxAwaitTimeMS = options.maxAwaitTimeMS ?? this.maxAwaitTimeMS;
|
|
784
|
+
|
|
710
785
|
return rawChangeStream(watchDb, pipeline, {
|
|
711
786
|
batchSize: options.batchSize ?? this.snapshotChunkLength,
|
|
712
|
-
maxAwaitTimeMS
|
|
787
|
+
maxAwaitTimeMS,
|
|
713
788
|
maxTimeMS: this.changeStreamTimeout,
|
|
714
789
|
signal: options.signal,
|
|
715
790
|
logger: this.logger,
|