@powersync/service-module-mongodb 0.18.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/dist/api/MongoRouteAPIAdapter.d.ts +2 -0
  3. package/dist/api/MongoRouteAPIAdapter.js +23 -32
  4. package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
  5. package/dist/common/SentinelLSN.d.ts +37 -0
  6. package/dist/common/SentinelLSN.js +59 -0
  7. package/dist/common/SentinelLSN.js.map +1 -0
  8. package/dist/replication/ChangeStream.d.ts +15 -0
  9. package/dist/replication/ChangeStream.js +105 -44
  10. package/dist/replication/ChangeStream.js.map +1 -1
  11. package/dist/replication/MongoRelation.d.ts +36 -1
  12. package/dist/replication/MongoRelation.js +102 -6
  13. package/dist/replication/MongoRelation.js.map +1 -1
  14. package/dist/replication/MongoSnapshotter.d.ts +9 -0
  15. package/dist/replication/MongoSnapshotter.js +108 -42
  16. package/dist/replication/MongoSnapshotter.js.map +1 -1
  17. package/dist/replication/RawChangeStream.d.ts +12 -1
  18. package/dist/replication/RawChangeStream.js +30 -7
  19. package/dist/replication/RawChangeStream.js.map +1 -1
  20. package/dist/replication/checkpoints/CheckpointImplementation.d.ts +132 -0
  21. package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
  22. package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
  23. package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +55 -0
  24. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +222 -0
  25. package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
  26. package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +24 -0
  27. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +113 -0
  28. package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
  29. package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
  30. package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
  31. package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
  32. package/dist/replication/replication-utils.d.ts +6 -0
  33. package/dist/replication/replication-utils.js +19 -2
  34. package/dist/replication/replication-utils.js.map +1 -1
  35. package/package.json +8 -8
  36. package/src/api/MongoRouteAPIAdapter.ts +26 -37
  37. package/src/common/SentinelLSN.ts +78 -0
  38. package/src/replication/ChangeStream.ts +118 -50
  39. package/src/replication/MongoRelation.ts +124 -14
  40. package/src/replication/MongoSnapshotter.ts +122 -47
  41. package/src/replication/RawChangeStream.ts +64 -28
  42. package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
  43. package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +267 -0
  44. package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +142 -0
  45. package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
  46. package/src/replication/replication-utils.ts +23 -2
  47. package/test/DOCUMENTDB_TESTING.md +115 -0
  48. package/test/src/DatabaseType.ts +25 -0
  49. package/test/src/change_stream.test.ts +97 -65
  50. package/test/src/change_stream_utils.ts +83 -7
  51. package/test/src/checkpoint_retry.test.ts +5 -2
  52. package/test/src/documentdb_helpers.test.ts +124 -0
  53. package/test/src/documentdb_mode.test.ts +1040 -0
  54. package/test/src/mongo_test.test.ts +15 -5
  55. package/test/src/raw_change_stream.test.ts +209 -125
  56. package/test/src/resume_token.test.ts +30 -0
  57. package/test/src/slow_tests.test.ts +4 -1
  58. package/test/src/test-timeouts.ts +23 -0
  59. package/test/src/util.ts +1 -2
  60. package/tsconfig.tsbuildinfo +1 -1
@@ -0,0 +1,167 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { Logger } from '@powersync/lib-services-framework';
3
+ import { ReplicationHeadCallback, storage } from '@powersync/service-core';
4
+
5
+ import { ProjectedChangeStreamDocument } from '../RawChangeStream.js';
6
+
7
+ /**
8
+ * Classification of an event on the `_powersync_checkpoints` collection.
9
+ *
10
+ * - 'standalone': source-wide checkpoint (write checkpoints, keepalive bumps,
11
+ * snapshot markers). Processed regardless of which process created it.
12
+ * - 'own-barrier': this stream's private batch barrier document.
13
+ * - 'foreign': another stream's barrier document — ignore.
14
+ */
15
+ export type CheckpointEventKind = 'standalone' | 'own-barrier' | 'foreign';
16
+
17
+ export interface StreamResumePosition {
18
+ resumeAfter: mongo.ResumeToken | null;
19
+ /**
20
+ * Legacy startAtOperationTime fallback. Only produced by the timestamp implementation,
21
+ * and only for old LSNs persisted without a resume token.
22
+ */
23
+ startAfter: mongo.Timestamp | null;
24
+ }
25
+
26
+ export interface CheckpointImplementationContext {
27
+ client: mongo.MongoClient;
28
+ db: mongo.Db;
29
+ checkpointStreamId: mongo.ObjectId;
30
+ logger: Logger;
31
+ }
32
+
33
+ /**
34
+ * Event-interpretation surface of a {@link CheckpointImplementation}. Every method
35
+ * operates on a single raw change event.
36
+ *
37
+ * Ordering contract: {@link observe} must be called before {@link lsn} for a given
38
+ * event, so the sentinel implementation's coordinate is current.
39
+ */
40
+ export interface CheckpointEventApi {
41
+ /**
42
+ * Classify a checkpoint-collection event and absorb any coordinate it
43
+ * carries (sentinel implementation). Must be called for every checkpoint event, in
44
+ * stream order.
45
+ */
46
+ observe(doc: ProjectedChangeStreamDocument): CheckpointEventKind;
47
+
48
+ /**
49
+ * Comparable LSN for committing/resuming at this event.
50
+ *
51
+ * Note this is not purely a function of `doc`. The timestamp implementation
52
+ * derives the whole LSN from the event (its coordinate is the event's
53
+ * clusterTime). The sentinel implementation pairs the *tracked* coordinate
54
+ * (updated by {@link observe}) with the event's resume token, because data
55
+ * events carry no coordinate of their own — hence the observe-before-lsn
56
+ * contract.
57
+ */
58
+ lsn(doc: ProjectedChangeStreamDocument): string;
59
+
60
+ /** Whether a barrier marker from {@link CheckpointImplementation.createBatchCheckpoint} is resolved by this event. */
61
+ resolvesBarrier(marker: string, doc: ProjectedChangeStreamDocument): boolean;
62
+ }
63
+
64
+ /**
65
+ * Strategy for producing and interpreting replication checkpoints.
66
+ *
67
+ * Two implementations exist:
68
+ *
69
+ * - {@link TimestampCheckpointImplementation}: standard MongoDB. The ordered LSN
70
+ * coordinate is the oplog clusterTime, which is unique per operation and
71
+ * parseable from resume tokens.
72
+ * - {@link SentinelCheckpointImplementation}: for sources without a usable clusterTime
73
+ * (DocumentDB, and technically usable on any MongoDB). The ordered coordinate
74
+ * is a shared monotonic counter document, observed through the change
75
+ * stream itself.
76
+ *
77
+ * An implementation instance belongs to a single ChangeStream (or API adapter) and may
78
+ * hold per-stream coordinate state; call {@link seedPosition} at the start of
79
+ * each streaming loop.
80
+ */
81
+ export interface CheckpointImplementation {
82
+ /** LSN representing "before any data". */
83
+ readonly zeroLsn: string;
84
+
85
+ /** Parse a stored LSN into change stream resume options. Pure. */
86
+ parseResumePosition(lsn: string): StreamResumePosition;
87
+
88
+ /** Reset/seed the implementation's coordinate state for a new stream loop. */
89
+ seedPosition(lsn: string | null): void;
90
+
91
+ /** Log the resume position at the start of a streaming loop. */
92
+ logResume(lsn: string): void;
93
+
94
+ /**
95
+ * Create a source-wide consistency checkpoint (snapshot boundaries,
96
+ * `no_checkpoint_before` markers). Returns a comparable LSN.
97
+ */
98
+ createStandaloneCheckpoint(): Promise<string>;
99
+
100
+ /**
101
+ * Create a batch barrier for this stream. Returns a marker that is resolved
102
+ * by a later change stream event via {@link CheckpointEventApi.resolvesBarrier}.
103
+ */
104
+ createBatchCheckpoint(): Promise<string>;
105
+
106
+ /**
107
+ * Create the first batch barrier for snapshot-LSN acquisition and return the
108
+ * LSN to open the change stream from, or null to open from "now".
109
+ *
110
+ * The timestamp implementation's barrier marker is itself a comparable LSN, so
111
+ * the stream resumes from it. The sentinel implementation's marker is opaque
112
+ * (content-matched) and carries no resume position, so it returns null and the
113
+ * stream opens from the current point.
114
+ */
115
+ createFirstBarrier(): Promise<string | null>;
116
+
117
+ /**
118
+ * Idle keepalive for an empty change stream batch. May persist a checkpoint
119
+ * directly (timestamp implementation) or nudge the source so that a later event
120
+ * commits (sentinel implementation).
121
+ */
122
+ keepalive(batch: storage.BucketStorageBatch, resumeToken: mongo.ResumeToken): Promise<void>;
123
+
124
+ /**
125
+ * Build a comparable LSN from a bare batch-level resume token (no change
126
+ * event). Used for the per-batch `setResumeLsn` progress marker.
127
+ *
128
+ * The timestamp implementation parses the timestamp embedded in the token.
129
+ * The sentinel implementation pairs the token with the current coordinate.
130
+ */
131
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken): string;
132
+
133
+ /**
134
+ * Source-side replication head for write checkpoints. The LSN passed to the
135
+ * callback must compare at or below any LSN committed after the caller's
136
+ * preceding writes.
137
+ */
138
+ createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
139
+
140
+ /** Event-interpretation methods, all operating on a single raw change event. */
141
+ readonly event: CheckpointEventApi;
142
+
143
+ /** Filter for clearing the checkpoints collection on startup. */
144
+ readonly checkpointClearFilter: mongo.Filter<mongo.Document>;
145
+ }
146
+
147
+ /**
148
+ * Extract the event timestamp: clusterTime when present, otherwise wallTime
149
+ * truncated to second precision.
150
+ */
151
+ export function getEventTimestamp(changeDocument: ProjectedChangeStreamDocument): mongo.Timestamp {
152
+ if (changeDocument.clusterTime) {
153
+ return changeDocument.clusterTime;
154
+ }
155
+ const wallTime = (changeDocument as any).wallTime as Date | undefined;
156
+ if (wallTime != null) {
157
+ return mongo.Timestamp.fromBits(0, Math.floor(wallTime.getTime() / 1000));
158
+ }
159
+ throw new Error('Change event has neither clusterTime nor wallTime');
160
+ }
161
+
162
+ export function getCheckpointId(doc: ProjectedChangeStreamDocument): string | mongo.ObjectId | null {
163
+ if (!('documentKey' in doc)) {
164
+ return null;
165
+ }
166
+ return doc.documentKey._id as string | mongo.ObjectId;
167
+ }
@@ -0,0 +1,267 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { ReplicationHeadCallback, storage } from '@powersync/service-core';
3
+ import { JSONBig } from '@powersync/service-jsonbig';
4
+ import { SentinelLSN } from '../../common/SentinelLSN.js';
5
+ import { ChangeStreamInvalidatedError } from '../ChangeStream.js';
6
+ import { createSentinelCheckpointLsn, SENTINEL_CHECKPOINT_ID } from '../MongoRelation.js';
7
+ import { ProjectedChangeStreamDocument } from '../RawChangeStream.js';
8
+ import {
9
+ CheckpointEventApi,
10
+ CheckpointEventKind,
11
+ CheckpointImplementation,
12
+ CheckpointImplementationContext,
13
+ getCheckpointId,
14
+ StreamResumePosition
15
+ } from './CheckpointImplementation.js';
16
+
17
+ /**
18
+ * Sentinel checkpoint implementation, used for sources without a usable clusterTime
19
+ * (DocumentDB). The ordered LSN coordinate is a single shared sentinel checkpoint
20
+ * counter ({@link SENTINEL_CHECKPOINT_ID}), observed through the change stream:
21
+ *
22
+ * - Batch checkpoints advance the global counter and stamp it with this stream's
23
+ * id, so the stream recognises its own private barriers by content (stream_id
24
+ * + counter) rather than by cross-document event ordering.
25
+ * - Standalone bumps (write checkpoint heads, snapshot markers) advance the same
26
+ * counter with a null stream_id; every stream observes these as the global
27
+ * coordinate.
28
+ * - Keepalives advance the counter without persisting; the bump's own event
29
+ * flows through the stream and commits via the standalone handling,
30
+ * preserving the "commit only what the stream has seen" barrier property.
31
+ *
32
+ * See docs/documentdb/documentdb-lsn-sentinel-checkpoints.md for the full design.
33
+ */
34
+ export class SentinelCheckpointImplementation implements CheckpointImplementation {
35
+ readonly zeroLsn = SentinelLSN.ZERO.comparable;
36
+
37
+ /**
38
+ * The highest global sentinel value proven so far, merged monotonically from
39
+ * the resume seed, standalone events and embedded barrier values. Needed
40
+ * for lsnFromResumeToken: per-batch resume markers carry only a resume token,
41
+ * so they need the latest observed coordinate paired in.
42
+ */
43
+ private position = SentinelLSN.ZERO.sentinel;
44
+
45
+ constructor(private context: CheckpointImplementationContext) {}
46
+
47
+ parseResumePosition(lsn: string): StreamResumePosition {
48
+ const parsed = SentinelLSN.fromSerialized(lsn);
49
+ return { resumeAfter: parsed.resumeToken ?? null, startAfter: null };
50
+ }
51
+
52
+ seedPosition(lsn: string | null): void {
53
+ this.position = lsn == null ? SentinelLSN.ZERO.sentinel : SentinelLSN.fromSerialized(lsn).sentinel;
54
+ }
55
+
56
+ logResume(lsn: string): void {
57
+ const parsed = SentinelLSN.fromSerialized(lsn);
58
+ this.context.logger.info(`Resume streaming at sentinel ${parsed.sentinel} / ${parsed}`);
59
+ }
60
+
61
+ async createStandaloneCheckpoint(): Promise<string> {
62
+ return createSentinelCheckpointLsn(this.context.client, this.context.db);
63
+ }
64
+
65
+ async createBatchCheckpoint(): Promise<string> {
66
+ // Advance the shared sentinel checkpoint — the global source-database
67
+ // coordinate. It must be shared so the LSN domain survives new
68
+ // ChangeStream instances and new sync rules.
69
+ // This advance is associated with the change stream in order to track stream
70
+ // barriers. The returned LSN (counter only, no resume token) is the barrier
71
+ // marker matched by resolvesBarrier.
72
+ return createSentinelCheckpointLsn(this.context.client, this.context.db, this.context.checkpointStreamId);
73
+ }
74
+
75
+ async createFirstBarrier(): Promise<string | null> {
76
+ // The barrier marker is an opaque content-matched marker, not a resume
77
+ // position, so there is no LSN to open the stream from: a fresh DocumentDB
78
+ // stream opens from "now" and the snapshot loop re-creates barriers until
79
+ // one is observed.
80
+ await this.createBatchCheckpoint();
81
+ return null;
82
+ }
83
+
84
+ async keepalive(_batch: storage.BucketStorageBatch, _resumeToken: mongo.ResumeToken): Promise<void> {
85
+ // Advance the shared sentinel, but do not persist a checkpoint here. The
86
+ // bump is stamped with this stream's id, so its own change event flows
87
+ // through the stream as an own-barrier event and is committed by the
88
+ // own-barrier handling, which advances the ordered LSN prefix and refreshes
89
+ // the resume token (using the event's own token).
90
+ //
91
+ // Why bump at all: with an unchanged sentinel, LSN comparison against the
92
+ // previously persisted LSN falls to the opaque base64 token suffix, which
93
+ // is not lexicographically meaningful — storage would silently reject
94
+ // roughly half of plain token refreshes, leaving the persisted resume
95
+ // token to go stale on an idle stream. (The timestamp implementation avoids this
96
+ // because its keepalive timestamp is parsed from the resume token itself,
97
+ // so the ordered prefix always advances with the token.)
98
+ //
99
+ // Why not persist immediately: writes that landed after this empty batch
100
+ // was read — including a write checkpoint head — would be covered by the
101
+ // persisted LSN before the stream has processed them, allowing write
102
+ // checkpoints to resolve before the corresponding data is replicated.
103
+ // Committing only when the bump's event is observed preserves the
104
+ // "commit only what the stream has seen" barrier property.
105
+ await createSentinelCheckpointLsn(this.context.client, this.context.db, this.context.checkpointStreamId);
106
+ this.context.logger.info(
107
+ `Idle change stream (sentinel implementation). Bumped sentinel to advance the checkpoint.`
108
+ );
109
+ }
110
+
111
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken): string {
112
+ // Pair the bare token with the current coordinate. The coordinate is
113
+ // frozen between checkpoint events, so a per-batch resume marker only
114
+ // advances the token; that is all resumption needs (see the design doc).
115
+ return new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable;
116
+ }
117
+
118
+ async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
119
+ const head = await createSentinelCheckpointLsn(this.context.client, this.context.db);
120
+ const result = await callback(head);
121
+ // Create another bump to ensure movement after the reported head. This
122
+ // covers the race where the head's own event is committed before the
123
+ // write checkpoint document is stored.
124
+ // Note that this checkpoint should not be associated with a change stream Id.
125
+ await createSentinelCheckpointLsn(this.context.client, this.context.db);
126
+ return result;
127
+ }
128
+
129
+ readonly event: CheckpointEventApi = {
130
+ observe: (doc) => {
131
+ const checkpointId = getCheckpointId(doc);
132
+ if (checkpointId != SENTINEL_CHECKPOINT_ID) {
133
+ // The sentinel implementation only uses the SENTINEL_CHECKPOINT_ID
134
+ // document; anything else (including the timestamp impl's standalone
135
+ // document) is foreign.
136
+ return 'foreign';
137
+ }
138
+ const fullDoc = deserializeFullDocument(doc);
139
+ if (fullDoc == null) {
140
+ // An insert/update/replace on our sentinel document must carry a
141
+ // post-image. A missing fullDocument means the document was deleted or
142
+ // replaced out from under us, which destroys the coordinate; invalidate
143
+ // so replication restarts clean instead of stalling.
144
+ throw new ChangeStreamInvalidatedError(
145
+ 'Sentinel checkpoint event has no fullDocument — cannot read the sentinel',
146
+ new Error(`Unexpected ${doc.operationType} event on the sentinel checkpoint document`)
147
+ );
148
+ }
149
+ const streamId = fullDoc.stream_id;
150
+
151
+ let kind: CheckpointEventKind;
152
+ if (streamId == null) {
153
+ // No stream_id: a standalone bump (write checkpoint head, snapshot
154
+ // marker). Every stream tracks these as the global coordinate.
155
+ kind = 'standalone';
156
+ } else if (this.context.checkpointStreamId.equals(streamId)) {
157
+ // Stamped with our id: one of our own private batch/keepalive barriers.
158
+ kind = 'own-barrier';
159
+ } else {
160
+ // Another stream's private barrier — ignore.
161
+ return 'foreign';
162
+ }
163
+
164
+ // Both kinds carry the global coordinate in `i`; keep our position current.
165
+ this.mergePosition(this.readSentinel(fullDoc));
166
+ return kind;
167
+ },
168
+
169
+ lsn: (doc) => {
170
+ const fullDoc = deserializeFullDocument(doc);
171
+ if (fullDoc == null) {
172
+ throw new ChangeStreamInvalidatedError(
173
+ 'Sentinel checkpoint event has no fullDocument — cannot read the sentinel',
174
+ new Error(`Unexpected ${doc.operationType} event on the sentinel checkpoint document`)
175
+ );
176
+ }
177
+
178
+ // Checkpoint events carry the global coordinate in `i`; pair that exact
179
+ // coordinate with this event's resume token. Plain data-batch resume
180
+ // markers use lsnFromResumeToken, which pairs the token with the tracked
181
+ // position instead.
182
+ return new SentinelLSN({
183
+ sentinel: this.readSentinel(fullDoc),
184
+ resume_token: doc._id
185
+ }).comparable;
186
+ },
187
+
188
+ resolvesBarrier: (marker, doc) => {
189
+ const fullDoc = deserializeFullDocument(doc);
190
+ if (fullDoc == null) {
191
+ this.context.logger.warn('Checkpoint event missing fullDocument — cannot match sentinel barrier');
192
+ return false;
193
+ }
194
+ const parsed = SentinelLSN.fromSerialized(marker);
195
+ return this.context.checkpointStreamId.equals(fullDoc.stream_id) && fullDoc.i >= parsed.sentinel;
196
+ }
197
+ };
198
+
199
+ // The sentinel checkpoint document must survive restarts (hence the $ne
200
+ // below — the startup cleanup deletes every other checkpoint document). Its
201
+ // counter is the globally-ordered component of every committed LSN, including
202
+ // write checkpoint heads. Deleting it would re-seed the counter on the next
203
+ // upsert, risking moving the LSN coordinate system backwards: new commits
204
+ // could compare below the persisted last_checkpoint_lsn (so storage rejects
205
+ // them via checkpointBlocked and the checkpoint stalls), and new write
206
+ // checkpoint heads could resolve against old, higher committed LSNs before
207
+ // their data has actually replicated.
208
+ //
209
+ // Note: this only protects against our own startup cleanup. The global
210
+ // LSN coordinate still lives in a user-visible collection, so a consumer
211
+ // can delete it in their source database. Dropping the whole checkpoints
212
+ // collection is detected (the streaming loop invalidates the stream on the
213
+ // collection drop event). Deleting just this document is mitigated by
214
+ // createSentinelCheckpointLsn seeding re-created counters at the current
215
+ // epoch seconds, so the coordinate jumps forward instead of resetting
216
+ // below already-committed LSNs.
217
+ readonly checkpointClearFilter: mongo.Filter<mongo.Document> = { _id: { $ne: SENTINEL_CHECKPOINT_ID } as any };
218
+
219
+ /** Never move the position backwards — replayed or reordered events must not regress the coordinate. */
220
+ private mergePosition(observed: bigint) {
221
+ if (observed > this.position) {
222
+ this.position = observed;
223
+ }
224
+ }
225
+
226
+ /**
227
+ * Read the sentinel counter from a sentinel checkpoint post-image.
228
+ *
229
+ * The `i` field is required: every sentinel write `$inc`s it. If it is
230
+ * missing, the document was replaced externally, which destroys the LSN
231
+ * coordinate system (a re-created counter restarts below already-committed
232
+ * LSNs). Invalidate the stream so replication restarts from scratch instead
233
+ * of stalling against a broken coordinate.
234
+ */
235
+ private readSentinel(fullDoc: mongo.Document): bigint {
236
+ if (fullDoc.i == null) {
237
+ throw new ChangeStreamInvalidatedError(
238
+ 'Sentinel checkpoint document has no `i` field — cannot read the sentinel',
239
+ // JSONBig.stringify, since the post-image is deserialized with useBigInt64.
240
+ new Error(`Sentinel checkpoint document: ${JSONBig.stringify(fullDoc)}`)
241
+ );
242
+ }
243
+ return fullDoc.i;
244
+ }
245
+ }
246
+
247
+ /**
248
+ * A change event with the decoded `fullDocument` post-image memoized on it, so
249
+ * a consumer that reads it more than once — e.g. observe() then resolvesBarrier()
250
+ * for the same event — decodes the buffer only once. `undefined` until first
251
+ * decoded; `null` when there is no fullDocument.
252
+ */
253
+ type MemoizedChangeStreamDocument = ProjectedChangeStreamDocument & {
254
+ parsedFullDocument?: mongo.Document | null;
255
+ };
256
+
257
+ function deserializeFullDocument(doc: ProjectedChangeStreamDocument): mongo.Document | null {
258
+ const memo = doc as MemoizedChangeStreamDocument;
259
+ if (memo.parsedFullDocument !== undefined) {
260
+ return memo.parsedFullDocument;
261
+ }
262
+ const fullDocument = 'fullDocument' in doc ? doc.fullDocument : null;
263
+ // fullDocument is a raw BSON Buffer from parseChangeDocument.
264
+ const parsed = fullDocument ? mongo.BSON.deserialize(fullDocument as Buffer, { useBigInt64: true }) : null;
265
+ memo.parsedFullDocument = parsed;
266
+ return parsed;
267
+ }
@@ -0,0 +1,142 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
3
+ import { ReplicationHeadCallback, storage } from '@powersync/service-core';
4
+ import { MongoLSN } from '../../common/MongoLSN.js';
5
+ import { createCheckpoint, SENTINEL_CHECKPOINT_ID, STANDALONE_CHECKPOINT_ID } from '../MongoRelation.js';
6
+ import { CHECKPOINTS_COLLECTION, timestampToDate } from '../replication-utils.js';
7
+ import {
8
+ CheckpointEventApi,
9
+ CheckpointImplementation,
10
+ CheckpointImplementationContext,
11
+ getCheckpointId,
12
+ getEventTimestamp,
13
+ StreamResumePosition
14
+ } from './CheckpointImplementation.js';
15
+
16
+ /**
17
+ * Standard MongoDB checkpoint implementation. The ordered LSN coordinate is the oplog
18
+ * clusterTime — unique per operation, monotonic, and parseable from resume
19
+ * tokens. Barriers and event LSNs are plain comparable LSN strings.
20
+ */
21
+ export class TimestampCheckpointImplementation implements CheckpointImplementation {
22
+ readonly zeroLsn = MongoLSN.ZERO.comparable;
23
+
24
+ constructor(private context: CheckpointImplementationContext) {}
25
+
26
+ parseResumePosition(lsn: string): StreamResumePosition {
27
+ const parsed = MongoLSN.fromSerialized(lsn);
28
+ return { resumeAfter: parsed.resumeToken ?? null, startAfter: parsed.timestamp };
29
+ }
30
+
31
+ seedPosition(_lsn: string | null): void {
32
+ // The coordinate comes from each event's clusterTime; no state to seed.
33
+ }
34
+
35
+ logResume(lsn: string): void {
36
+ const parsed = MongoLSN.fromSerialized(lsn);
37
+ // It is normal for this to be a minute or two old when there is a low volume
38
+ // of ChangeStream events.
39
+ const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(parsed.timestamp).getTime()) / 1000);
40
+ this.context.logger.info(
41
+ `Resume streaming at ${parsed.timestamp.inspect()} / ${parsed} | Token age: ${tokenAgeSeconds}s`
42
+ );
43
+ }
44
+
45
+ async createStandaloneCheckpoint(): Promise<string> {
46
+ return createCheckpoint(this.context.db, STANDALONE_CHECKPOINT_ID);
47
+ }
48
+
49
+ async createBatchCheckpoint(): Promise<string> {
50
+ return createCheckpoint(this.context.db, this.context.checkpointStreamId);
51
+ }
52
+
53
+ async createFirstBarrier(): Promise<string | null> {
54
+ // The barrier marker is a comparable LSN; resume the snapshot stream from it.
55
+ return this.createBatchCheckpoint();
56
+ }
57
+
58
+ async keepalive(batch: storage.BucketStorageBatch, resumeToken: mongo.ResumeToken): Promise<void> {
59
+ // Parse the timestamp from the resume token. The ordered LSN prefix
60
+ // advances together with the token, so persisting is always safe.
61
+ const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(resumeToken);
62
+ await batch.keepalive(lsn);
63
+ // Log the token update. This helps as a general "replication is still active" message in the logs.
64
+ // This token would typically be around 10s behind.
65
+ this.context.logger.info(
66
+ `Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`
67
+ );
68
+ }
69
+
70
+ lsnFromResumeToken(resumeToken: mongo.ResumeToken): string {
71
+ // The timestamp is embedded in the resume token.
72
+ return MongoLSN.fromResumeToken(resumeToken).comparable;
73
+ }
74
+
75
+ async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
76
+ const session = this.context.client.startSession();
77
+ try {
78
+ await this.context.db.command({ hello: 1 }, { session });
79
+ const head = session.clusterTime?.clusterTime;
80
+ if (head == null) {
81
+ throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
82
+ }
83
+
84
+ const r = await callback(new MongoLSN({ timestamp: head }).comparable);
85
+
86
+ // Trigger a change on the changestream, so that the write checkpoint
87
+ // is processed without waiting for other writes.
88
+ await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
89
+ {
90
+ _id: STANDALONE_CHECKPOINT_ID as any
91
+ },
92
+ {
93
+ $inc: { i: 1 }
94
+ },
95
+ {
96
+ upsert: true,
97
+ returnDocument: 'after',
98
+ session
99
+ }
100
+ );
101
+ const time = session.operationTime!;
102
+ if (time == null) {
103
+ throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
104
+ } else if (time.lt(head)) {
105
+ throw new ServiceAssertionError(`operationTime must be > clusterTime`);
106
+ }
107
+
108
+ return r;
109
+ } finally {
110
+ await session.endSession();
111
+ }
112
+ }
113
+
114
+ readonly event: CheckpointEventApi = {
115
+ observe: (doc) => {
116
+ const checkpointId = getCheckpointId(doc);
117
+ if (checkpointId == null || checkpointId == SENTINEL_CHECKPOINT_ID) {
118
+ return 'foreign';
119
+ }
120
+ // The STANDALONE_CHECKPOINT_ID is only used for the TimestampCheckpointImplementation
121
+ if (checkpointId == STANDALONE_CHECKPOINT_ID) {
122
+ return 'standalone';
123
+ }
124
+ return this.context.checkpointStreamId.equals(checkpointId) ? 'own-barrier' : 'foreign';
125
+ },
126
+
127
+ lsn: (doc) => {
128
+ return new MongoLSN({
129
+ timestamp: getEventTimestamp(doc),
130
+ resume_token: doc._id
131
+ }).comparable;
132
+ },
133
+
134
+ resolvesBarrier: (marker, doc) => {
135
+ // Barrier markers are comparable LSNs in this implementation.
136
+ return this.event.lsn(doc) >= marker;
137
+ }
138
+ };
139
+
140
+ // It's safe to clear the entire _powersync_checkpoints collection in this mode.
141
+ readonly checkpointClearFilter: mongo.Filter<mongo.Document> = {};
142
+ }
@@ -0,0 +1,14 @@
1
+ import { CheckpointImplementation, CheckpointImplementationContext } from './CheckpointImplementation.js';
2
+ import { SentinelCheckpointImplementation } from './SentinelCheckpointImplementation.js';
3
+ import { TimestampCheckpointImplementation } from './TimestampCheckpointImplementation.js';
4
+
5
+ /**
6
+ * Select the checkpoint implementation for a source: sentinel-based for DocumentDB
7
+ * DB, clusterTime-based for standard MongoDB.
8
+ */
9
+ export function createCheckpointImplementation(
10
+ isDocumentDb: boolean,
11
+ context: CheckpointImplementationContext
12
+ ): CheckpointImplementation {
13
+ return isDocumentDb ? new SentinelCheckpointImplementation(context) : new TimestampCheckpointImplementation(context);
14
+ }
@@ -1,3 +1,4 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
1
2
  import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
2
3
  import * as bson from 'bson';
3
4
  import { PostImagesOption } from '../types/types.js';
@@ -7,16 +8,36 @@ export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
7
8
 
8
9
  const REQUIRED_CHECKPOINT_PERMISSIONS = ['find', 'insert', 'update', 'remove', 'changeStream', 'createCollection'];
9
10
 
11
+ /**
12
+ * Whether a `hello` response indicates Azure DocumentDB (formerly Azure Cosmos
13
+ * DB for MongoDB vCore), which reports `documentdb_versions` in the `internal`
14
+ * section.
15
+ */
16
+ function isDocumentDbHello(hello: mongo.Document): boolean {
17
+ return hello.internal?.documentdb_versions != null;
18
+ }
19
+
20
+ /**
21
+ * Detect whether the connected server is DocumentDB. DocumentDB lacks usable
22
+ * clusterTime/operationTime and uses the sentinel checkpoint implementation.
23
+ */
24
+ export async function detectDocumentDb(db: mongo.Db): Promise<boolean> {
25
+ const hello = await db.command({ hello: 1 });
26
+ return isDocumentDbHello(hello);
27
+ }
28
+
10
29
  export async function checkSourceConfiguration(connectionManager: MongoManager): Promise<void> {
11
30
  const db = connectionManager.db;
12
31
 
13
32
  const hello = await db.command({ hello: 1 });
14
- if (hello.msg == 'isdbgrid') {
33
+ const isDocumentDb = isDocumentDbHello(hello);
34
+
35
+ if (hello.msg == 'isdbgrid' && !isDocumentDb) {
15
36
  throw new ServiceError(
16
37
  ErrorCode.PSYNC_S1341,
17
38
  'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'
18
39
  );
19
- } else if (hello.setName == null) {
40
+ } else if (hello.setName == null && !isDocumentDb) {
20
41
  throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
21
42
  }
22
43