@powersync/service-module-mongodb 0.0.0-dev-20260601140105 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -5
- package/dist/api/MongoRouteAPIAdapter.d.ts +4 -2
- package/dist/api/MongoRouteAPIAdapter.js +23 -32
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/common/SentinelLSN.d.ts +37 -0
- package/dist/common/SentinelLSN.js +59 -0
- package/dist/common/SentinelLSN.js.map +1 -0
- package/dist/replication/ChangeStream.d.ts +16 -0
- package/dist/replication/ChangeStream.js +143 -64
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/ChangeStreamReplicator.js +1 -1
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/dist/replication/MongoRelation.d.ts +36 -1
- package/dist/replication/MongoRelation.js +133 -18
- package/dist/replication/MongoRelation.js.map +1 -1
- package/dist/replication/MongoSnapshotter.d.ts +9 -1
- package/dist/replication/MongoSnapshotter.js +128 -66
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +12 -1
- package/dist/replication/RawChangeStream.js +23 -10
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/checkpoints/CheckpointImplementation.d.ts +132 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +55 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +224 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +24 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +115 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
- package/dist/replication/replication-utils.d.ts +6 -0
- package/dist/replication/replication-utils.js +19 -2
- package/dist/replication/replication-utils.js.map +1 -1
- package/dist/types/types.d.ts +2 -2
- package/package.json +11 -21
- package/src/api/MongoRouteAPIAdapter.ts +28 -39
- package/src/common/SentinelLSN.ts +78 -0
- package/src/replication/ChangeStream.ts +159 -73
- package/src/replication/ChangeStreamReplicationJob.ts +1 -1
- package/src/replication/ChangeStreamReplicator.ts +1 -1
- package/src/replication/MongoRelation.ts +153 -21
- package/src/replication/MongoSnapshotter.ts +147 -72
- package/src/replication/RawChangeStream.ts +54 -32
- package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
- package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +269 -0
- package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +144 -0
- package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
- package/src/replication/replication-utils.ts +23 -2
- package/test/DOCUMENTDB_TESTING.md +115 -0
- package/test/src/DatabaseType.ts +25 -0
- package/test/src/change_stream.test.ts +107 -71
- package/test/src/change_stream_utils.ts +117 -29
- package/test/src/checkpoint_retry.test.ts +134 -0
- package/test/src/documentdb_helpers.test.ts +124 -0
- package/test/src/documentdb_mode.test.ts +1040 -0
- package/test/src/mongo_test.test.ts +15 -5
- package/test/src/raw_change_stream.test.ts +209 -125
- package/test/src/resume.test.ts +2 -2
- package/test/src/resume_token.test.ts +30 -0
- package/test/src/slow_tests.test.ts +4 -1
- package/test/src/test-timeouts.ts +23 -0
- package/test/src/util.ts +3 -3
- package/tsconfig.json +1 -4
- package/tsconfig.scripts.json +1 -2
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { JSONBig } from '@powersync/service-jsonbig';
|
|
3
|
+
import { SentinelLSN } from '../../common/SentinelLSN.js';
|
|
4
|
+
import { ChangeStreamInvalidatedError } from '../ChangeStream.js';
|
|
5
|
+
import { createSentinelCheckpointLsn, SENTINEL_CHECKPOINT_ID } from '../MongoRelation.js';
|
|
6
|
+
import { getCheckpointId } from './CheckpointImplementation.js';
|
|
7
|
+
/**
|
|
8
|
+
* Sentinel checkpoint implementation, used for sources without a usable clusterTime
|
|
9
|
+
* (DocumentDB). The ordered LSN coordinate is a single shared sentinel checkpoint
|
|
10
|
+
* counter ({@link SENTINEL_CHECKPOINT_ID}), observed through the change stream:
|
|
11
|
+
*
|
|
12
|
+
* - Batch checkpoints advance the global counter and stamp it with this stream's
|
|
13
|
+
* id, so the stream recognises its own private barriers by content (stream_id
|
|
14
|
+
* + counter) rather than by cross-document event ordering.
|
|
15
|
+
* - Standalone bumps (write checkpoint heads, snapshot markers) advance the same
|
|
16
|
+
* counter with a null stream_id; every stream observes these as the global
|
|
17
|
+
* coordinate.
|
|
18
|
+
* - Keepalives advance the counter without persisting; the bump's own event
|
|
19
|
+
* flows through the stream and commits via the standalone handling,
|
|
20
|
+
* preserving the "commit only what the stream has seen" barrier property.
|
|
21
|
+
*
|
|
22
|
+
* See docs/documentdb/documentdb-lsn-sentinel-checkpoints.md for the full design.
|
|
23
|
+
*/
|
|
24
|
+
export class SentinelCheckpointImplementation {
|
|
25
|
+
context;
|
|
26
|
+
zeroLsn = SentinelLSN.ZERO.comparable;
|
|
27
|
+
/**
|
|
28
|
+
* The highest global sentinel value proven so far, merged monotonically from
|
|
29
|
+
* the resume seed, standalone events and embedded barrier values. Needed
|
|
30
|
+
* for lsnFromResumeToken: per-batch resume markers carry only a resume token,
|
|
31
|
+
* so they need the latest observed coordinate paired in.
|
|
32
|
+
*/
|
|
33
|
+
position = SentinelLSN.ZERO.sentinel;
|
|
34
|
+
constructor(context) {
|
|
35
|
+
this.context = context;
|
|
36
|
+
}
|
|
37
|
+
parseResumePosition(lsn) {
|
|
38
|
+
const parsed = SentinelLSN.fromSerialized(lsn);
|
|
39
|
+
return { resumeAfter: parsed.resumeToken ?? null, startAfter: null };
|
|
40
|
+
}
|
|
41
|
+
seedPosition(lsn) {
|
|
42
|
+
this.position = lsn == null ? SentinelLSN.ZERO.sentinel : SentinelLSN.fromSerialized(lsn).sentinel;
|
|
43
|
+
}
|
|
44
|
+
logResume(lsn) {
|
|
45
|
+
const parsed = SentinelLSN.fromSerialized(lsn);
|
|
46
|
+
this.context.logger.info(`Resume streaming at sentinel ${parsed.sentinel} / ${parsed}`);
|
|
47
|
+
}
|
|
48
|
+
async createStandaloneCheckpoint() {
|
|
49
|
+
return createSentinelCheckpointLsn(this.context.client, this.context.db);
|
|
50
|
+
}
|
|
51
|
+
async createBatchCheckpoint() {
|
|
52
|
+
// Advance the shared sentinel checkpoint — the global source-database
|
|
53
|
+
// coordinate. It must be shared so the LSN domain survives new
|
|
54
|
+
// ChangeStream instances and new sync rules.
|
|
55
|
+
// This advance is associated with the change stream in order to track stream
|
|
56
|
+
// barriers. The returned LSN (counter only, no resume token) is the barrier
|
|
57
|
+
// marker matched by resolvesBarrier.
|
|
58
|
+
return createSentinelCheckpointLsn(this.context.client, this.context.db, this.context.checkpointStreamId);
|
|
59
|
+
}
|
|
60
|
+
async createFirstBarrier() {
|
|
61
|
+
// The barrier marker is an opaque content-matched marker, not a resume
|
|
62
|
+
// position, so there is no LSN to open the stream from: a fresh DocumentDB
|
|
63
|
+
// stream opens from "now" and the snapshot loop re-creates barriers until
|
|
64
|
+
// one is observed.
|
|
65
|
+
await this.createBatchCheckpoint();
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
async keepalive(_batch, _resumeToken) {
|
|
69
|
+
// Advance the shared sentinel, but do not persist a checkpoint here. The
|
|
70
|
+
// bump is stamped with this stream's id, so its own change event flows
|
|
71
|
+
// through the stream as an own-barrier event and is committed by the
|
|
72
|
+
// own-barrier handling, which advances the ordered LSN prefix and refreshes
|
|
73
|
+
// the resume token (using the event's own token).
|
|
74
|
+
//
|
|
75
|
+
// Why bump at all: with an unchanged sentinel, LSN comparison against the
|
|
76
|
+
// previously persisted LSN falls to the opaque base64 token suffix, which
|
|
77
|
+
// is not lexicographically meaningful — storage would silently reject
|
|
78
|
+
// roughly half of plain token refreshes, leaving the persisted resume
|
|
79
|
+
// token to go stale on an idle stream. (The timestamp implementation avoids this
|
|
80
|
+
// because its keepalive timestamp is parsed from the resume token itself,
|
|
81
|
+
// so the ordered prefix always advances with the token.)
|
|
82
|
+
//
|
|
83
|
+
// Why not persist immediately: writes that landed after this empty batch
|
|
84
|
+
// was read — including a write checkpoint head — would be covered by the
|
|
85
|
+
// persisted LSN before the stream has processed them, allowing write
|
|
86
|
+
// checkpoints to resolve before the corresponding data is replicated.
|
|
87
|
+
// Committing only when the bump's event is observed preserves the
|
|
88
|
+
// "commit only what the stream has seen" barrier property.
|
|
89
|
+
await createSentinelCheckpointLsn(this.context.client, this.context.db, this.context.checkpointStreamId);
|
|
90
|
+
this.context.logger.info(`Idle change stream (sentinel implementation). Bumped sentinel to advance the checkpoint.`);
|
|
91
|
+
}
|
|
92
|
+
lsnFromResumeToken(resumeToken) {
|
|
93
|
+
// Pair the bare token with the current coordinate. The coordinate is
|
|
94
|
+
// frozen between checkpoint events, so a per-batch resume marker only
|
|
95
|
+
// advances the token; that is all resumption needs (see the design doc).
|
|
96
|
+
return new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable;
|
|
97
|
+
}
|
|
98
|
+
async createReplicationHead(callback) {
|
|
99
|
+
const head = await createSentinelCheckpointLsn(this.context.client, this.context.db);
|
|
100
|
+
const { response, shouldAdvance } = await callback(head);
|
|
101
|
+
if (shouldAdvance) {
|
|
102
|
+
// Create another bump to ensure movement after the reported head. This
|
|
103
|
+
// covers the race where the head's own event is committed before the
|
|
104
|
+
// write checkpoint document is stored.
|
|
105
|
+
// Note that this checkpoint should not be associated with a change stream Id.
|
|
106
|
+
await createSentinelCheckpointLsn(this.context.client, this.context.db);
|
|
107
|
+
}
|
|
108
|
+
return response;
|
|
109
|
+
}
|
|
110
|
+
event = {
|
|
111
|
+
observe: (doc) => {
|
|
112
|
+
const checkpointId = getCheckpointId(doc);
|
|
113
|
+
if (checkpointId != SENTINEL_CHECKPOINT_ID) {
|
|
114
|
+
// The sentinel implementation only uses the SENTINEL_CHECKPOINT_ID
|
|
115
|
+
// document; anything else (including the timestamp impl's standalone
|
|
116
|
+
// document) is foreign.
|
|
117
|
+
return 'foreign';
|
|
118
|
+
}
|
|
119
|
+
const fullDoc = deserializeFullDocument(doc);
|
|
120
|
+
if (fullDoc == null) {
|
|
121
|
+
// An insert/update/replace on our sentinel document must carry a
|
|
122
|
+
// post-image. A missing fullDocument means the document was deleted or
|
|
123
|
+
// replaced out from under us, which destroys the coordinate; invalidate
|
|
124
|
+
// so replication restarts clean instead of stalling.
|
|
125
|
+
throw new ChangeStreamInvalidatedError('Sentinel checkpoint event has no fullDocument — cannot read the sentinel', new Error(`Unexpected ${doc.operationType} event on the sentinel checkpoint document`));
|
|
126
|
+
}
|
|
127
|
+
const streamId = fullDoc.stream_id;
|
|
128
|
+
let kind;
|
|
129
|
+
if (streamId == null) {
|
|
130
|
+
// No stream_id: a standalone bump (write checkpoint head, snapshot
|
|
131
|
+
// marker). Every stream tracks these as the global coordinate.
|
|
132
|
+
kind = 'standalone';
|
|
133
|
+
}
|
|
134
|
+
else if (this.context.checkpointStreamId.equals(streamId)) {
|
|
135
|
+
// Stamped with our id: one of our own private batch/keepalive barriers.
|
|
136
|
+
kind = 'own-barrier';
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
// Another stream's private barrier — ignore.
|
|
140
|
+
return 'foreign';
|
|
141
|
+
}
|
|
142
|
+
// Both kinds carry the global coordinate in `i`; keep our position current.
|
|
143
|
+
this.mergePosition(this.readSentinel(fullDoc));
|
|
144
|
+
return kind;
|
|
145
|
+
},
|
|
146
|
+
lsn: (doc) => {
|
|
147
|
+
const fullDoc = deserializeFullDocument(doc);
|
|
148
|
+
if (fullDoc == null) {
|
|
149
|
+
throw new ChangeStreamInvalidatedError('Sentinel checkpoint event has no fullDocument — cannot read the sentinel', new Error(`Unexpected ${doc.operationType} event on the sentinel checkpoint document`));
|
|
150
|
+
}
|
|
151
|
+
// Checkpoint events carry the global coordinate in `i`; pair that exact
|
|
152
|
+
// coordinate with this event's resume token. Plain data-batch resume
|
|
153
|
+
// markers use lsnFromResumeToken, which pairs the token with the tracked
|
|
154
|
+
// position instead.
|
|
155
|
+
return new SentinelLSN({
|
|
156
|
+
sentinel: this.readSentinel(fullDoc),
|
|
157
|
+
resume_token: doc._id
|
|
158
|
+
}).comparable;
|
|
159
|
+
},
|
|
160
|
+
resolvesBarrier: (marker, doc) => {
|
|
161
|
+
const fullDoc = deserializeFullDocument(doc);
|
|
162
|
+
if (fullDoc == null) {
|
|
163
|
+
this.context.logger.warn('Checkpoint event missing fullDocument — cannot match sentinel barrier');
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
const parsed = SentinelLSN.fromSerialized(marker);
|
|
167
|
+
return this.context.checkpointStreamId.equals(fullDoc.stream_id) && fullDoc.i >= parsed.sentinel;
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
// The sentinel checkpoint document must survive restarts (hence the $ne
|
|
171
|
+
// below — the startup cleanup deletes every other checkpoint document). Its
|
|
172
|
+
// counter is the globally-ordered component of every committed LSN, including
|
|
173
|
+
// write checkpoint heads. Deleting it would re-seed the counter on the next
|
|
174
|
+
// upsert, risking moving the LSN coordinate system backwards: new commits
|
|
175
|
+
// could compare below the persisted last_checkpoint_lsn (so storage rejects
|
|
176
|
+
// them via checkpointBlocked and the checkpoint stalls), and new write
|
|
177
|
+
// checkpoint heads could resolve against old, higher committed LSNs before
|
|
178
|
+
// their data has actually replicated.
|
|
179
|
+
//
|
|
180
|
+
// Note: this only protects against our own startup cleanup. The global
|
|
181
|
+
// LSN coordinate still lives in a user-visible collection, so a consumer
|
|
182
|
+
// can delete it in their source database. Dropping the whole checkpoints
|
|
183
|
+
// collection is detected (the streaming loop invalidates the stream on the
|
|
184
|
+
// collection drop event). Deleting just this document is mitigated by
|
|
185
|
+
// createSentinelCheckpointLsn seeding re-created counters at the current
|
|
186
|
+
// epoch seconds, so the coordinate jumps forward instead of resetting
|
|
187
|
+
// below already-committed LSNs.
|
|
188
|
+
checkpointClearFilter = { _id: { $ne: SENTINEL_CHECKPOINT_ID } };
|
|
189
|
+
/** Never move the position backwards — replayed or reordered events must not regress the coordinate. */
|
|
190
|
+
mergePosition(observed) {
|
|
191
|
+
if (observed > this.position) {
|
|
192
|
+
this.position = observed;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Read the sentinel counter from a sentinel checkpoint post-image.
|
|
197
|
+
*
|
|
198
|
+
* The `i` field is required: every sentinel write `$inc`s it. If it is
|
|
199
|
+
* missing, the document was replaced externally, which destroys the LSN
|
|
200
|
+
* coordinate system (a re-created counter restarts below already-committed
|
|
201
|
+
* LSNs). Invalidate the stream so replication restarts from scratch instead
|
|
202
|
+
* of stalling against a broken coordinate.
|
|
203
|
+
*/
|
|
204
|
+
readSentinel(fullDoc) {
|
|
205
|
+
if (fullDoc.i == null) {
|
|
206
|
+
throw new ChangeStreamInvalidatedError('Sentinel checkpoint document has no `i` field — cannot read the sentinel',
|
|
207
|
+
// JSONBig.stringify, since the post-image is deserialized with useBigInt64.
|
|
208
|
+
new Error(`Sentinel checkpoint document: ${JSONBig.stringify(fullDoc)}`));
|
|
209
|
+
}
|
|
210
|
+
return fullDoc.i;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
function deserializeFullDocument(doc) {
|
|
214
|
+
const memo = doc;
|
|
215
|
+
if (memo.parsedFullDocument !== undefined) {
|
|
216
|
+
return memo.parsedFullDocument;
|
|
217
|
+
}
|
|
218
|
+
const fullDocument = 'fullDocument' in doc ? doc.fullDocument : null;
|
|
219
|
+
// fullDocument is a raw BSON Buffer from parseChangeDocument.
|
|
220
|
+
const parsed = fullDocument ? mongo.BSON.deserialize(fullDocument, { useBigInt64: true }) : null;
|
|
221
|
+
memo.parsedFullDocument = parsed;
|
|
222
|
+
return parsed;
|
|
223
|
+
}
|
|
224
|
+
//# sourceMappingURL=SentinelCheckpointImplementation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SentinelCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/SentinelCheckpointImplementation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EAKL,eAAe,EAEhB,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,gCAAgC;IAWvB;IAVX,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;IAE/C;;;;;OAKG;IACK,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;IAE7C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACvE,CAAC;IAED,YAAY,CAAC,GAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrG,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,sEAAsE;QACtE,+DAA+D;QAC/D,6CAA6C;QAC7C,6EAA6E;QAC7E,4EAA4E;QAC5E,qCAAqC;QACrC,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5G,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,uEAAuE;QACvE,2EAA2E;QAC3E,0EAA0E;QAC1E,mBAAmB;QACnB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAkC,EAAE,YAA+B;QACjF,yEAAyE;QACzE,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,kDAAkD;QAClD,EAAE;QACF,0EAA0E;QAC1E,0EAA0E;QAC1E,sEAAsE;QACtE,sEAAsE;QACtE,iFAAiF;QACjF,0EAA0E;QAC1E,yDAAyD;QACzD,EAAE;QACF,yEAAyE;QACzE,yEAAyE;QACzE,qEAAqE;QACrE,sEAAsE;QACtE,kEAAkE;QAClE,2DAA2D;QAC3D,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACzG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,OAAO,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,UAAU,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,IAAI,GAAG,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACrF,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QACzD,IAAI,aAAa,EAAE,CAAC;YAClB,uEAAuE;YACvE,qEAAqE;YACrE,uCAAuC;YACvC,8EAA8E;YAC9E,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBAC3C,mEAAmE;gBACnE,qEAAqE;gBACrE,wBAAwB;gBACxB,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,iEAAiE;gBACjE,uEAAuE;gBACvE,wEAAwE;gBACxE,qDAAqD;gBACrD,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;YAEnC,IAAI,IAAyB,CAAC;YAC9B,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,mEAAmE;gBACnE,+DAA+D;gBAC/D,IAAI,GAAG,YAAY,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5D,wEAAwE;gBACxE,IAAI,GAAG,aAAa,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,6CAA6C;gBAC7C,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,4EAA4E;YAC5E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YAED,wEAAwE;YACxE,qEAAqE;YACrE,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,IAAI,WAAW,CAAC;gBACrB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;gBACpC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;gBAClG,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC;QACnG,CAAC;KACF,CAAC;IAEF,wEAAwE;IACxE,4EAA4E;IAC5E,8EAA8E;IAC9E,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAC5E,uEAAuE;IACvE,2EAA2E;IAC3E,sCAAsC;IACtC,EAAE;IACF,uEAAuE;IACvE,yEAAyE;IACzE,yEAAyE;IACzE,2EAA2E;IAC3E,sEAAsE;IACtE,yEAAyE;IACzE,sEAAsE;IACtE,gCAAgC;IACvB,qBAAqB,GAAiC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,sBAAsB,EAAS,EAAE,CAAC;IAE/G,wGAAwG;IAChG,aAAa,CAAC,QAAgB;QACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,YAAY,CAAC,OAAuB;QAC1C,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E;YAC1E,4EAA4E;YAC5E,IAAI,KAAK,CAAC,iCAAiC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CACzE,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC;IACnB,CAAC;CACF;AAYD,SAAS,uBAAuB,CAAC,GAAkC;IACjE,MAAM,IAAI,GAAG,GAAmC,CAAC;IACjD,IAAI,IAAI,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,YAAsB,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ReplicationHeadCallback, storage } from '@powersync/service-core';
|
|
3
|
+
import { CheckpointEventApi, CheckpointImplementation, CheckpointImplementationContext, StreamResumePosition } from './CheckpointImplementation.js';
|
|
4
|
+
/**
|
|
5
|
+
* Standard MongoDB checkpoint implementation. The ordered LSN coordinate is the oplog
|
|
6
|
+
* clusterTime — unique per operation, monotonic, and parseable from resume
|
|
7
|
+
* tokens. Barriers and event LSNs are plain comparable LSN strings.
|
|
8
|
+
*/
|
|
9
|
+
export declare class TimestampCheckpointImplementation implements CheckpointImplementation {
|
|
10
|
+
private context;
|
|
11
|
+
readonly zeroLsn: string;
|
|
12
|
+
constructor(context: CheckpointImplementationContext);
|
|
13
|
+
parseResumePosition(lsn: string): StreamResumePosition;
|
|
14
|
+
seedPosition(_lsn: string | null): void;
|
|
15
|
+
logResume(lsn: string): void;
|
|
16
|
+
createStandaloneCheckpoint(): Promise<string>;
|
|
17
|
+
createBatchCheckpoint(): Promise<string>;
|
|
18
|
+
createFirstBarrier(): Promise<string | null>;
|
|
19
|
+
keepalive(batch: storage.BucketStorageBatch, resumeToken: mongo.ResumeToken): Promise<void>;
|
|
20
|
+
lsnFromResumeToken(resumeToken: mongo.ResumeToken): string;
|
|
21
|
+
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
22
|
+
readonly event: CheckpointEventApi;
|
|
23
|
+
readonly checkpointClearFilter: mongo.Filter<mongo.Document>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { MongoLSN } from '../../common/MongoLSN.js';
|
|
3
|
+
import { createCheckpoint, SENTINEL_CHECKPOINT_ID, STANDALONE_CHECKPOINT_ID } from '../MongoRelation.js';
|
|
4
|
+
import { CHECKPOINTS_COLLECTION, timestampToDate } from '../replication-utils.js';
|
|
5
|
+
import { getCheckpointId, getEventTimestamp } from './CheckpointImplementation.js';
|
|
6
|
+
/**
|
|
7
|
+
* Standard MongoDB checkpoint implementation. The ordered LSN coordinate is the oplog
|
|
8
|
+
* clusterTime — unique per operation, monotonic, and parseable from resume
|
|
9
|
+
* tokens. Barriers and event LSNs are plain comparable LSN strings.
|
|
10
|
+
*/
|
|
11
|
+
export class TimestampCheckpointImplementation {
|
|
12
|
+
context;
|
|
13
|
+
zeroLsn = MongoLSN.ZERO.comparable;
|
|
14
|
+
constructor(context) {
|
|
15
|
+
this.context = context;
|
|
16
|
+
}
|
|
17
|
+
parseResumePosition(lsn) {
|
|
18
|
+
const parsed = MongoLSN.fromSerialized(lsn);
|
|
19
|
+
return { resumeAfter: parsed.resumeToken ?? null, startAfter: parsed.timestamp };
|
|
20
|
+
}
|
|
21
|
+
seedPosition(_lsn) {
|
|
22
|
+
// The coordinate comes from each event's clusterTime; no state to seed.
|
|
23
|
+
}
|
|
24
|
+
logResume(lsn) {
|
|
25
|
+
const parsed = MongoLSN.fromSerialized(lsn);
|
|
26
|
+
// It is normal for this to be a minute or two old when there is a low volume
|
|
27
|
+
// of ChangeStream events.
|
|
28
|
+
const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(parsed.timestamp).getTime()) / 1000);
|
|
29
|
+
this.context.logger.info(`Resume streaming at ${parsed.timestamp.inspect()} / ${parsed} | Token age: ${tokenAgeSeconds}s`);
|
|
30
|
+
}
|
|
31
|
+
async createStandaloneCheckpoint() {
|
|
32
|
+
return createCheckpoint(this.context.db, STANDALONE_CHECKPOINT_ID);
|
|
33
|
+
}
|
|
34
|
+
async createBatchCheckpoint() {
|
|
35
|
+
return createCheckpoint(this.context.db, this.context.checkpointStreamId);
|
|
36
|
+
}
|
|
37
|
+
async createFirstBarrier() {
|
|
38
|
+
// The barrier marker is a comparable LSN; resume the snapshot stream from it.
|
|
39
|
+
return this.createBatchCheckpoint();
|
|
40
|
+
}
|
|
41
|
+
async keepalive(batch, resumeToken) {
|
|
42
|
+
// Parse the timestamp from the resume token. The ordered LSN prefix
|
|
43
|
+
// advances together with the token, so persisting is always safe.
|
|
44
|
+
const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(resumeToken);
|
|
45
|
+
await batch.keepalive(lsn);
|
|
46
|
+
// Log the token update. This helps as a general "replication is still active" message in the logs.
|
|
47
|
+
// This token would typically be around 10s behind.
|
|
48
|
+
this.context.logger.info(`Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`);
|
|
49
|
+
}
|
|
50
|
+
lsnFromResumeToken(resumeToken) {
|
|
51
|
+
// The timestamp is embedded in the resume token.
|
|
52
|
+
return MongoLSN.fromResumeToken(resumeToken).comparable;
|
|
53
|
+
}
|
|
54
|
+
async createReplicationHead(callback) {
|
|
55
|
+
const session = this.context.client.startSession();
|
|
56
|
+
try {
|
|
57
|
+
await this.context.db.command({ hello: 1 }, { session });
|
|
58
|
+
const head = session.clusterTime?.clusterTime;
|
|
59
|
+
if (head == null) {
|
|
60
|
+
throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
|
|
61
|
+
}
|
|
62
|
+
const { response, shouldAdvance } = await callback(new MongoLSN({ timestamp: head }).comparable);
|
|
63
|
+
if (shouldAdvance) {
|
|
64
|
+
// Trigger a change on the changestream, so that the write checkpoint
|
|
65
|
+
// is processed without waiting for other writes.
|
|
66
|
+
await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate({
|
|
67
|
+
_id: STANDALONE_CHECKPOINT_ID
|
|
68
|
+
}, {
|
|
69
|
+
$inc: { i: 1 }
|
|
70
|
+
}, {
|
|
71
|
+
upsert: true,
|
|
72
|
+
returnDocument: 'after',
|
|
73
|
+
session
|
|
74
|
+
});
|
|
75
|
+
const time = session.operationTime;
|
|
76
|
+
if (time == null) {
|
|
77
|
+
throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
|
|
78
|
+
}
|
|
79
|
+
else if (time.lt(head)) {
|
|
80
|
+
throw new ServiceAssertionError(`operationTime must be > clusterTime`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return response;
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await session.endSession();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
event = {
|
|
90
|
+
observe: (doc) => {
|
|
91
|
+
const checkpointId = getCheckpointId(doc);
|
|
92
|
+
if (checkpointId == null || checkpointId == SENTINEL_CHECKPOINT_ID) {
|
|
93
|
+
return 'foreign';
|
|
94
|
+
}
|
|
95
|
+
// The STANDALONE_CHECKPOINT_ID is only used for the TimestampCheckpointImplementation
|
|
96
|
+
if (checkpointId == STANDALONE_CHECKPOINT_ID) {
|
|
97
|
+
return 'standalone';
|
|
98
|
+
}
|
|
99
|
+
return this.context.checkpointStreamId.equals(checkpointId) ? 'own-barrier' : 'foreign';
|
|
100
|
+
},
|
|
101
|
+
lsn: (doc) => {
|
|
102
|
+
return new MongoLSN({
|
|
103
|
+
timestamp: getEventTimestamp(doc),
|
|
104
|
+
resume_token: doc._id
|
|
105
|
+
}).comparable;
|
|
106
|
+
},
|
|
107
|
+
resolvesBarrier: (marker, doc) => {
|
|
108
|
+
// Barrier markers are comparable LSNs in this implementation.
|
|
109
|
+
return this.event.lsn(doc) >= marker;
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
// It's safe to clear the entire _powersync_checkpoints collection in this mode.
|
|
113
|
+
checkpointClearFilter = {};
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=TimestampCheckpointImplementation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TimestampCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/TimestampCheckpointImplementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACzG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAIL,eAAe,EACf,iBAAiB,EAElB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,iCAAiC;IAGxB;IAFX,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;IAE5C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;IACnF,CAAC;IAED,YAAY,CAAC,IAAmB;QAC9B,wEAAwE;IAC1E,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,6EAA6E;QAC7E,0BAA0B;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACtG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,uBAAuB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,MAAM,iBAAiB,eAAe,GAAG,CACjG,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,8EAA8E;QAC9E,OAAO,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiC,EAAE,WAA8B;QAC/E,oEAAoE;QACpE,kEAAkE;QAClE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,mGAAmG;QACnG,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,iDAAiD,eAAe,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,iDAAiD;QACjD,OAAO,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC;YAC9C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;YACpF,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAEjG,IAAI,aAAa,EAAE,CAAC;gBAClB,qEAAqE;gBACrE,iDAAiD;gBACjD,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,gBAAgB,CACvE;oBACE,GAAG,EAAE,wBAA+B;iBACrC,EACD;oBACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;iBACf,EACD;oBACE,MAAM,EAAE,IAAI;oBACZ,cAAc,EAAE,OAAO;oBACvB,OAAO;iBACR,CACF,CAAC;gBACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;gBACpC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,MAAM,IAAI,qBAAqB,CAAC,kDAAkD,CAAC,CAAC;gBACtF,CAAC;qBAAM,IAAI,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBACnE,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,sFAAsF;YACtF,IAAI,YAAY,IAAI,wBAAwB,EAAE,CAAC;gBAC7C,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1F,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,OAAO,IAAI,QAAQ,CAAC;gBAClB,SAAS,EAAE,iBAAiB,CAAC,GAAG,CAAC;gBACjC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,8DAA8D;YAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;QACvC,CAAC;KACF,CAAC;IAEF,gFAAgF;IACvE,qBAAqB,GAAiC,EAAE,CAAC;CACnE"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { CheckpointImplementation, CheckpointImplementationContext } from './CheckpointImplementation.js';
|
|
2
|
+
/**
|
|
3
|
+
* Select the checkpoint implementation for a source: sentinel-based for DocumentDB
|
|
4
|
+
* DB, clusterTime-based for standard MongoDB.
|
|
5
|
+
*/
|
|
6
|
+
export declare function createCheckpointImplementation(isDocumentDb: boolean, context: CheckpointImplementationContext): CheckpointImplementation;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { SentinelCheckpointImplementation } from './SentinelCheckpointImplementation.js';
|
|
2
|
+
import { TimestampCheckpointImplementation } from './TimestampCheckpointImplementation.js';
|
|
3
|
+
/**
|
|
4
|
+
* Select the checkpoint implementation for a source: sentinel-based for DocumentDB
|
|
5
|
+
* DB, clusterTime-based for standard MongoDB.
|
|
6
|
+
*/
|
|
7
|
+
export function createCheckpointImplementation(isDocumentDb, context) {
|
|
8
|
+
return isDocumentDb ? new SentinelCheckpointImplementation(context) : new TimestampCheckpointImplementation(context);
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=create-checkpoint-implementation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create-checkpoint-implementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/create-checkpoint-implementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AACzF,OAAO,EAAE,iCAAiC,EAAE,MAAM,wCAAwC,CAAC;AAE3F;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,YAAqB,EACrB,OAAwC;IAExC,OAAO,YAAY,CAAC,CAAC,CAAC,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,iCAAiC,CAAC,OAAO,CAAC,CAAC;AACvH,CAAC"}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import * as bson from 'bson';
|
|
2
3
|
import { MongoManager } from './MongoManager.js';
|
|
3
4
|
export declare const CHECKPOINTS_COLLECTION = "_powersync_checkpoints";
|
|
5
|
+
/**
|
|
6
|
+
* Detect whether the connected server is DocumentDB. DocumentDB lacks usable
|
|
7
|
+
* clusterTime/operationTime and uses the sentinel checkpoint implementation.
|
|
8
|
+
*/
|
|
9
|
+
export declare function detectDocumentDb(db: mongo.Db): Promise<boolean>;
|
|
4
10
|
export declare function checkSourceConfiguration(connectionManager: MongoManager): Promise<void>;
|
|
5
11
|
export declare function timestampToDate(timestamp: bson.Timestamp): Date;
|
|
@@ -2,13 +2,30 @@ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
|
2
2
|
import { PostImagesOption } from '../types/types.js';
|
|
3
3
|
export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
|
|
4
4
|
const REQUIRED_CHECKPOINT_PERMISSIONS = ['find', 'insert', 'update', 'remove', 'changeStream', 'createCollection'];
|
|
5
|
+
/**
|
|
6
|
+
* Whether a `hello` response indicates Azure DocumentDB (formerly Azure Cosmos
|
|
7
|
+
* DB for MongoDB vCore), which reports `documentdb_versions` in the `internal`
|
|
8
|
+
* section.
|
|
9
|
+
*/
|
|
10
|
+
function isDocumentDbHello(hello) {
|
|
11
|
+
return hello.internal?.documentdb_versions != null;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Detect whether the connected server is DocumentDB. DocumentDB lacks usable
|
|
15
|
+
* clusterTime/operationTime and uses the sentinel checkpoint implementation.
|
|
16
|
+
*/
|
|
17
|
+
export async function detectDocumentDb(db) {
|
|
18
|
+
const hello = await db.command({ hello: 1 });
|
|
19
|
+
return isDocumentDbHello(hello);
|
|
20
|
+
}
|
|
5
21
|
export async function checkSourceConfiguration(connectionManager) {
|
|
6
22
|
const db = connectionManager.db;
|
|
7
23
|
const hello = await db.command({ hello: 1 });
|
|
8
|
-
|
|
24
|
+
const isDocumentDb = isDocumentDbHello(hello);
|
|
25
|
+
if (hello.msg == 'isdbgrid' && !isDocumentDb) {
|
|
9
26
|
throw new ServiceError(ErrorCode.PSYNC_S1341, 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
|
|
10
27
|
}
|
|
11
|
-
else if (hello.setName == null) {
|
|
28
|
+
else if (hello.setName == null && !isDocumentDb) {
|
|
12
29
|
throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
|
|
13
30
|
}
|
|
14
31
|
// https://www.mongodb.com/docs/manual/reference/command/connectionStatus/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replication-utils.js","sourceRoot":"","sources":["../../src/replication/replication-utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replication-utils.js","sourceRoot":"","sources":["../../src/replication/replication-utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGrD,MAAM,CAAC,MAAM,sBAAsB,GAAG,wBAAwB,CAAC;AAE/D,MAAM,+BAA+B,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,kBAAkB,CAAC,CAAC;AAEnH;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,KAAqB;IAC9C,OAAO,KAAK,CAAC,QAAQ,EAAE,mBAAmB,IAAI,IAAI,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAY;IACjD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,iBAA+B;IAC5E,MAAM,EAAE,GAAG,iBAAiB,CAAC,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAE9C,IAAI,KAAK,CAAC,GAAG,IAAI,UAAU,IAAI,CAAC,YAAY,EAAE,CAAC;QAC7C,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,0FAA0F,CAC3F,CAAC;IACJ,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAClD,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,oEAAoE,CAAC,CAAC;IACtH,CAAC;IAED,0EAA0E;IAC1E,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,EAAE,2BAG5C,CAAC;IACJ,IAAI,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC3C,IAAI,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7C,IAAI,WAAW,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,YAAY,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QACvG,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,IAAI,sBAAsB,IAAI,CAAC,CAAC,QAAQ,EAAE,UAAU,IAAI,EAAE,CACvF,CAAC;QAEF,KAAK,IAAI,CAAC,IAAI,aAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACxB,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,KAAK,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;YAC1B,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACxB,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,wBAAwB,GAAG,+BAA+B,CAAC,MAAM,CACrE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAC5C,CAAC;QACF,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,YAAY,IAAI,sBAAsB,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,2CAA2C,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,QAAQ,IAAI,CACrI,CAAC;QACJ,CAAC;QAED,IAAI,iBAAiB,CAAC,OAAO,CAAC,UAAU,IAAI,gBAAgB,CAAC,cAAc,EAAE,CAAC;YAC5E,kEAAkE;YAClE,+FAA+F;YAC/F,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,mEAAmE,EAAE,CAAC,YAAY,gDAAgD,CACnI,CAAC;YACJ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,2EAA2E,EAAE,CAAC,YAAY,IAAI,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,2BAA2B;QAC3B,kGAAkG;QAElG,gGAAgG;QAChG,MAAM,EAAE;aACL,eAAe,CACd;YACE,IAAI,EAAE,sBAAsB;SAC7B,EACD,EAAE,QAAQ,EAAE,KAAK,EAAE,CACpB;aACA,OAAO,EAAE,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAyB;IACvD,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE,GAAG,IAAI,CAAC,CAAC;AAC1D,CAAC"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -51,8 +51,8 @@ export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
|
|
|
51
51
|
tag?: string | undefined;
|
|
52
52
|
debug_api?: boolean | undefined;
|
|
53
53
|
} & {
|
|
54
|
-
uri: string;
|
|
55
54
|
type: "mongodb";
|
|
55
|
+
uri: string;
|
|
56
56
|
database?: string | undefined;
|
|
57
57
|
username?: string | undefined;
|
|
58
58
|
password?: string | undefined;
|
|
@@ -63,8 +63,8 @@ export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
|
|
|
63
63
|
tag?: string | undefined;
|
|
64
64
|
debug_api?: boolean | undefined;
|
|
65
65
|
} & {
|
|
66
|
-
uri: string;
|
|
67
66
|
type: "mongodb";
|
|
67
|
+
uri: string;
|
|
68
68
|
database?: string | undefined;
|
|
69
69
|
username?: string | undefined;
|
|
70
70
|
password?: string | undefined;
|
package/package.json
CHANGED
|
@@ -1,41 +1,31 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-mongodb",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"
|
|
5
|
-
"version": "0.0.0-dev-20260601140105",
|
|
6
|
-
"main": "dist/index.js",
|
|
4
|
+
"version": "0.0.0-dev-20260803144812",
|
|
7
5
|
"license": "FSL-1.1-ALv2",
|
|
8
6
|
"type": "module",
|
|
9
7
|
"publishConfig": {
|
|
10
8
|
"access": "public"
|
|
11
9
|
},
|
|
12
10
|
"exports": {
|
|
13
|
-
".":
|
|
14
|
-
|
|
15
|
-
"require": "./dist/index.js",
|
|
16
|
-
"default": "./dist/index.js"
|
|
17
|
-
},
|
|
18
|
-
"./types": {
|
|
19
|
-
"import": "./dist/types/types.js",
|
|
20
|
-
"require": "./dist/types/types.js",
|
|
21
|
-
"default": "./dist/types/types.js"
|
|
22
|
-
}
|
|
11
|
+
".": "./dist/index.js",
|
|
12
|
+
"./types": "./dist/types/types.js"
|
|
23
13
|
},
|
|
24
14
|
"dependencies": {
|
|
25
15
|
"bson": "^6.10.4",
|
|
26
16
|
"ts-codec": "^1.3.0",
|
|
27
17
|
"uuid": "^14.0.0",
|
|
28
|
-
"@powersync/lib-service-mongodb": "0.0.0-dev-
|
|
29
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
30
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
18
|
+
"@powersync/lib-service-mongodb": "0.0.0-dev-20260803144812",
|
|
19
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20260803144812",
|
|
20
|
+
"@powersync/service-core": "0.0.0-dev-20260803144812",
|
|
31
21
|
"@powersync/service-jsonbig": "0.17.13",
|
|
32
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
33
|
-
"@powersync/service-types": "0.
|
|
22
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20260803144812",
|
|
23
|
+
"@powersync/service-types": "0.0.0-dev-20260803144812"
|
|
34
24
|
},
|
|
35
25
|
"devDependencies": {
|
|
36
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
37
|
-
"@powersync/service-module-mongodb-storage": "0.0.0-dev-
|
|
38
|
-
"@powersync/service-module-postgres-storage": "0.0.0-dev-
|
|
26
|
+
"@powersync/service-core-tests": "0.0.0-dev-20260803144812",
|
|
27
|
+
"@powersync/service-module-mongodb-storage": "0.0.0-dev-20260803144812",
|
|
28
|
+
"@powersync/service-module-postgres-storage": "0.0.0-dev-20260803144812"
|
|
39
29
|
},
|
|
40
30
|
"scripts": {
|
|
41
31
|
"build": "tsc -b",
|