@powersync/service-module-mongodb 0.18.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +2 -0
- package/dist/api/MongoRouteAPIAdapter.js +23 -32
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/common/SentinelLSN.d.ts +37 -0
- package/dist/common/SentinelLSN.js +59 -0
- package/dist/common/SentinelLSN.js.map +1 -0
- package/dist/module/MongoModule.js +2 -1
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +23 -0
- package/dist/replication/ChangeStream.js +167 -47
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +2 -1
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/MongoRelation.d.ts +36 -1
- package/dist/replication/MongoRelation.js +102 -6
- package/dist/replication/MongoRelation.js.map +1 -1
- package/dist/replication/MongoSnapshotter.d.ts +9 -0
- package/dist/replication/MongoSnapshotter.js +113 -42
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +12 -1
- package/dist/replication/RawChangeStream.js +23 -10
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/checkpoints/CheckpointImplementation.d.ts +135 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js +22 -0
- package/dist/replication/checkpoints/CheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +58 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +224 -0
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +27 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +116 -0
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.d.ts +6 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js +10 -0
- package/dist/replication/checkpoints/create-checkpoint-implementation.js.map +1 -0
- package/dist/replication/replication-utils.d.ts +6 -0
- package/dist/replication/replication-utils.js +19 -2
- package/dist/replication/replication-utils.js.map +1 -1
- package/dist/types/types.d.ts +5 -0
- package/dist/types/types.js +20 -2
- package/dist/types/types.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MongoRouteAPIAdapter.ts +26 -37
- package/src/common/SentinelLSN.ts +78 -0
- package/src/module/MongoModule.ts +2 -1
- package/src/replication/ChangeStream.ts +184 -55
- package/src/replication/ChangeStreamReplicationJob.ts +2 -1
- package/src/replication/MongoRelation.ts +124 -14
- package/src/replication/MongoSnapshotter.ts +132 -47
- package/src/replication/RawChangeStream.ts +54 -32
- package/src/replication/checkpoints/CheckpointImplementation.ts +167 -0
- package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +269 -0
- package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +145 -0
- package/src/replication/checkpoints/create-checkpoint-implementation.ts +14 -0
- package/src/replication/replication-utils.ts +23 -2
- package/src/types/types.ts +27 -2
- package/test/DOCUMENTDB_TESTING.md +115 -0
- package/test/src/DatabaseType.ts +25 -0
- package/test/src/change_stream.test.ts +97 -65
- package/test/src/change_stream_utils.ts +99 -11
- package/test/src/checkpoint_retry.test.ts +5 -2
- package/test/src/config.test.ts +34 -0
- package/test/src/documentdb_helpers.test.ts +124 -0
- package/test/src/documentdb_mode.test.ts +1040 -0
- package/test/src/mongo_test.test.ts +15 -5
- package/test/src/raw_change_stream.test.ts +209 -125
- package/test/src/resume_token.test.ts +30 -0
- package/test/src/slow_tests.test.ts +4 -1
- package/test/src/test-timeouts.ts +23 -0
- package/test/src/util.ts +1 -2
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { MongoLSN } from '../../common/MongoLSN.js';
|
|
3
|
+
import { createCheckpoint, SENTINEL_CHECKPOINT_ID, STANDALONE_CHECKPOINT_ID } from '../MongoRelation.js';
|
|
4
|
+
import { CHECKPOINTS_COLLECTION, timestampToDate } from '../replication-utils.js';
|
|
5
|
+
import { getCheckpointId, getEventTimestamp } from './CheckpointImplementation.js';
|
|
6
|
+
/**
|
|
7
|
+
* Standard MongoDB checkpoint implementation. The ordered LSN coordinate is the oplog
|
|
8
|
+
* clusterTime — unique per operation, monotonic, and parseable from resume
|
|
9
|
+
* tokens. Barriers and event LSNs are plain comparable LSN strings.
|
|
10
|
+
*/
|
|
11
|
+
export class TimestampCheckpointImplementation {
|
|
12
|
+
context;
|
|
13
|
+
zeroLsn = MongoLSN.ZERO.comparable;
|
|
14
|
+
constructor(context) {
|
|
15
|
+
this.context = context;
|
|
16
|
+
}
|
|
17
|
+
parseResumePosition(lsn) {
|
|
18
|
+
const parsed = MongoLSN.fromSerialized(lsn);
|
|
19
|
+
return { resumeAfter: parsed.resumeToken ?? null, startAfter: parsed.timestamp };
|
|
20
|
+
}
|
|
21
|
+
seedPosition(_lsn) {
|
|
22
|
+
// The coordinate comes from each event's clusterTime; no state to seed.
|
|
23
|
+
}
|
|
24
|
+
logResume(lsn) {
|
|
25
|
+
const parsed = MongoLSN.fromSerialized(lsn);
|
|
26
|
+
// It is normal for this to be a minute or two old when there is a low volume
|
|
27
|
+
// of ChangeStream events.
|
|
28
|
+
const tokenAgeSeconds = Math.round((Date.now() - timestampToDate(parsed.timestamp).getTime()) / 1000);
|
|
29
|
+
this.context.logger.info(`Resume streaming at ${parsed.timestamp.inspect()} / ${parsed} | Token age: ${tokenAgeSeconds}s`);
|
|
30
|
+
}
|
|
31
|
+
async createStandaloneCheckpoint() {
|
|
32
|
+
return createCheckpoint(this.context.db, STANDALONE_CHECKPOINT_ID);
|
|
33
|
+
}
|
|
34
|
+
async createBatchCheckpoint() {
|
|
35
|
+
return createCheckpoint(this.context.db, this.context.checkpointStreamId);
|
|
36
|
+
}
|
|
37
|
+
async createFirstBarrier() {
|
|
38
|
+
// The barrier marker is a comparable LSN; resume the snapshot stream from it.
|
|
39
|
+
return this.createBatchCheckpoint();
|
|
40
|
+
}
|
|
41
|
+
async keepalive(batch, resumeToken) {
|
|
42
|
+
// Parse the timestamp from the resume token. The ordered LSN prefix
|
|
43
|
+
// advances together with the token, so persisting is always safe.
|
|
44
|
+
const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(resumeToken);
|
|
45
|
+
await batch.keepalive(lsn);
|
|
46
|
+
// Log the token update. This helps as a general "replication is still active" message in the logs.
|
|
47
|
+
// This token would typically be around 10s behind.
|
|
48
|
+
this.context.logger.info(`Idle change stream. Persisted resumeToken for ${timestampToDate(timestamp).toISOString()}`);
|
|
49
|
+
}
|
|
50
|
+
lsnFromResumeToken(resumeToken) {
|
|
51
|
+
// The timestamp is embedded in the resume token.
|
|
52
|
+
const lsn = MongoLSN.fromResumeToken(resumeToken);
|
|
53
|
+
return { lsn: lsn.comparable, timestamp: timestampToDate(lsn.timestamp) };
|
|
54
|
+
}
|
|
55
|
+
async createReplicationHead(callback) {
|
|
56
|
+
const session = this.context.client.startSession();
|
|
57
|
+
try {
|
|
58
|
+
await this.context.db.command({ hello: 1 }, { session });
|
|
59
|
+
const head = session.clusterTime?.clusterTime;
|
|
60
|
+
if (head == null) {
|
|
61
|
+
throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
|
|
62
|
+
}
|
|
63
|
+
const { response, shouldAdvance } = await callback(new MongoLSN({ timestamp: head }).comparable);
|
|
64
|
+
if (shouldAdvance) {
|
|
65
|
+
// Trigger a change on the changestream, so that the write checkpoint
|
|
66
|
+
// is processed without waiting for other writes.
|
|
67
|
+
await this.context.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate({
|
|
68
|
+
_id: STANDALONE_CHECKPOINT_ID
|
|
69
|
+
}, {
|
|
70
|
+
$inc: { i: 1 }
|
|
71
|
+
}, {
|
|
72
|
+
upsert: true,
|
|
73
|
+
returnDocument: 'after',
|
|
74
|
+
session
|
|
75
|
+
});
|
|
76
|
+
const time = session.operationTime;
|
|
77
|
+
if (time == null) {
|
|
78
|
+
throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
|
|
79
|
+
}
|
|
80
|
+
else if (time.lt(head)) {
|
|
81
|
+
throw new ServiceAssertionError(`operationTime must be > clusterTime`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return response;
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
await session.endSession();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
event = {
|
|
91
|
+
observe: (doc) => {
|
|
92
|
+
const checkpointId = getCheckpointId(doc);
|
|
93
|
+
if (checkpointId == null || checkpointId == SENTINEL_CHECKPOINT_ID) {
|
|
94
|
+
return 'foreign';
|
|
95
|
+
}
|
|
96
|
+
// The STANDALONE_CHECKPOINT_ID is only used for the TimestampCheckpointImplementation
|
|
97
|
+
if (checkpointId == STANDALONE_CHECKPOINT_ID) {
|
|
98
|
+
return 'standalone';
|
|
99
|
+
}
|
|
100
|
+
return this.context.checkpointStreamId.equals(checkpointId) ? 'own-barrier' : 'foreign';
|
|
101
|
+
},
|
|
102
|
+
lsn: (doc) => {
|
|
103
|
+
return new MongoLSN({
|
|
104
|
+
timestamp: getEventTimestamp(doc),
|
|
105
|
+
resume_token: doc._id
|
|
106
|
+
}).comparable;
|
|
107
|
+
},
|
|
108
|
+
resolvesBarrier: (marker, doc) => {
|
|
109
|
+
// Barrier markers are comparable LSNs in this implementation.
|
|
110
|
+
return this.event.lsn(doc) >= marker;
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
// It's safe to clear the entire _powersync_checkpoints collection in this mode.
|
|
114
|
+
checkpointClearFilter = {};
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=TimestampCheckpointImplementation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TimestampCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/TimestampCheckpointImplementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACzG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAIL,eAAe,EACf,iBAAiB,EAElB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,iCAAiC;IAGxB;IAFX,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;IAE5C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;IACnF,CAAC;IAED,YAAY,CAAC,IAAmB;QAC9B,wEAAwE;IAC1E,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,6EAA6E;QAC7E,0BAA0B;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACtG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,uBAAuB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,MAAM,iBAAiB,eAAe,GAAG,CACjG,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,8EAA8E;QAC9E,OAAO,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiC,EAAE,WAA8B;QAC/E,oEAAoE;QACpE,kEAAkE;QAClE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,mGAAmG;QACnG,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,iDAAiD,eAAe,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,iDAAiD;QACjD,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAClD,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC;YAC9C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;YACpF,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAEjG,IAAI,aAAa,EAAE,CAAC;gBAClB,qEAAqE;gBACrE,iDAAiD;gBACjD,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,gBAAgB,CACvE;oBACE,GAAG,EAAE,wBAA+B;iBACrC,EACD;oBACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;iBACf,EACD;oBACE,MAAM,EAAE,IAAI;oBACZ,cAAc,EAAE,OAAO;oBACvB,OAAO;iBACR,CACF,CAAC;gBACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;gBACpC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,MAAM,IAAI,qBAAqB,CAAC,kDAAkD,CAAC,CAAC;gBACtF,CAAC;qBAAM,IAAI,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBACnE,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,sFAAsF;YACtF,IAAI,YAAY,IAAI,wBAAwB,EAAE,CAAC;gBAC7C,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1F,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,OAAO,IAAI,QAAQ,CAAC;gBAClB,SAAS,EAAE,iBAAiB,CAAC,GAAG,CAAC;gBACjC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,8DAA8D;YAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;QACvC,CAAC;KACF,CAAC;IAEF,gFAAgF;IACvE,qBAAqB,GAAiC,EAAE,CAAC;CACnE"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { CheckpointImplementation, CheckpointImplementationContext } from './CheckpointImplementation.js';
|
|
2
|
+
/**
|
|
3
|
+
* Select the checkpoint implementation for a source: sentinel-based for DocumentDB
|
|
4
|
+
* DB, clusterTime-based for standard MongoDB.
|
|
5
|
+
*/
|
|
6
|
+
export declare function createCheckpointImplementation(isDocumentDb: boolean, context: CheckpointImplementationContext): CheckpointImplementation;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { SentinelCheckpointImplementation } from './SentinelCheckpointImplementation.js';
|
|
2
|
+
import { TimestampCheckpointImplementation } from './TimestampCheckpointImplementation.js';
|
|
3
|
+
/**
|
|
4
|
+
* Select the checkpoint implementation for a source: sentinel-based for DocumentDB
|
|
5
|
+
* DB, clusterTime-based for standard MongoDB.
|
|
6
|
+
*/
|
|
7
|
+
export function createCheckpointImplementation(isDocumentDb, context) {
|
|
8
|
+
return isDocumentDb ? new SentinelCheckpointImplementation(context) : new TimestampCheckpointImplementation(context);
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=create-checkpoint-implementation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create-checkpoint-implementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/create-checkpoint-implementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gCAAgC,EAAE,MAAM,uCAAuC,CAAC;AACzF,OAAO,EAAE,iCAAiC,EAAE,MAAM,wCAAwC,CAAC;AAE3F;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,YAAqB,EACrB,OAAwC;IAExC,OAAO,YAAY,CAAC,CAAC,CAAC,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,iCAAiC,CAAC,OAAO,CAAC,CAAC;AACvH,CAAC"}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import * as bson from 'bson';
|
|
2
3
|
import { MongoManager } from './MongoManager.js';
|
|
3
4
|
export declare const CHECKPOINTS_COLLECTION = "_powersync_checkpoints";
|
|
5
|
+
/**
|
|
6
|
+
* Detect whether the connected server is DocumentDB. DocumentDB lacks usable
|
|
7
|
+
* clusterTime/operationTime and uses the sentinel checkpoint implementation.
|
|
8
|
+
*/
|
|
9
|
+
export declare function detectDocumentDb(db: mongo.Db): Promise<boolean>;
|
|
4
10
|
export declare function checkSourceConfiguration(connectionManager: MongoManager): Promise<void>;
|
|
5
11
|
export declare function timestampToDate(timestamp: bson.Timestamp): Date;
|
|
@@ -2,13 +2,30 @@ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
|
2
2
|
import { PostImagesOption } from '../types/types.js';
|
|
3
3
|
export const CHECKPOINTS_COLLECTION = '_powersync_checkpoints';
|
|
4
4
|
const REQUIRED_CHECKPOINT_PERMISSIONS = ['find', 'insert', 'update', 'remove', 'changeStream', 'createCollection'];
|
|
5
|
+
/**
|
|
6
|
+
* Whether a `hello` response indicates Azure DocumentDB (formerly Azure Cosmos
|
|
7
|
+
* DB for MongoDB vCore), which reports `documentdb_versions` in the `internal`
|
|
8
|
+
* section.
|
|
9
|
+
*/
|
|
10
|
+
function isDocumentDbHello(hello) {
|
|
11
|
+
return hello.internal?.documentdb_versions != null;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Detect whether the connected server is DocumentDB. DocumentDB lacks usable
|
|
15
|
+
* clusterTime/operationTime and uses the sentinel checkpoint implementation.
|
|
16
|
+
*/
|
|
17
|
+
export async function detectDocumentDb(db) {
|
|
18
|
+
const hello = await db.command({ hello: 1 });
|
|
19
|
+
return isDocumentDbHello(hello);
|
|
20
|
+
}
|
|
5
21
|
export async function checkSourceConfiguration(connectionManager) {
|
|
6
22
|
const db = connectionManager.db;
|
|
7
23
|
const hello = await db.command({ hello: 1 });
|
|
8
|
-
|
|
24
|
+
const isDocumentDb = isDocumentDbHello(hello);
|
|
25
|
+
if (hello.msg == 'isdbgrid' && !isDocumentDb) {
|
|
9
26
|
throw new ServiceError(ErrorCode.PSYNC_S1341, 'Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).');
|
|
10
27
|
}
|
|
11
|
-
else if (hello.setName == null) {
|
|
28
|
+
else if (hello.setName == null && !isDocumentDb) {
|
|
12
29
|
throw new ServiceError(ErrorCode.PSYNC_S1342, 'Standalone MongoDB instances are not supported - use a replicaset.');
|
|
13
30
|
}
|
|
14
31
|
// https://www.mongodb.com/docs/manual/reference/command/connectionStatus/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replication-utils.js","sourceRoot":"","sources":["../../src/replication/replication-utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replication-utils.js","sourceRoot":"","sources":["../../src/replication/replication-utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGrD,MAAM,CAAC,MAAM,sBAAsB,GAAG,wBAAwB,CAAC;AAE/D,MAAM,+BAA+B,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,kBAAkB,CAAC,CAAC;AAEnH;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,KAAqB;IAC9C,OAAO,KAAK,CAAC,QAAQ,EAAE,mBAAmB,IAAI,IAAI,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAY;IACjD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,iBAA+B;IAC5E,MAAM,EAAE,GAAG,iBAAiB,CAAC,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAE9C,IAAI,KAAK,CAAC,GAAG,IAAI,UAAU,IAAI,CAAC,YAAY,EAAE,CAAC;QAC7C,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,0FAA0F,CAC3F,CAAC;IACJ,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAClD,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,oEAAoE,CAAC,CAAC;IACtH,CAAC;IAED,0EAA0E;IAC1E,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,EAAE,2BAG5C,CAAC;IACJ,IAAI,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC3C,IAAI,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7C,IAAI,WAAW,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,YAAY,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QACvG,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,IAAI,sBAAsB,IAAI,CAAC,CAAC,QAAQ,EAAE,UAAU,IAAI,EAAE,CACvF,CAAC;QAEF,KAAK,IAAI,CAAC,IAAI,aAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACxB,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,KAAK,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;YAC1B,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACxB,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,wBAAwB,GAAG,+BAA+B,CAAC,MAAM,CACrE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAC5C,CAAC;QACF,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,YAAY,IAAI,sBAAsB,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,2CAA2C,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,QAAQ,IAAI,CACrI,CAAC;QACJ,CAAC;QAED,IAAI,iBAAiB,CAAC,OAAO,CAAC,UAAU,IAAI,gBAAgB,CAAC,cAAc,EAAE,CAAC;YAC5E,kEAAkE;YAClE,+FAA+F;YAC/F,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,mEAAmE,EAAE,CAAC,YAAY,gDAAgD,CACnI,CAAC;YACJ,CAAC;QACH,CAAC;QACD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,2EAA2E,EAAE,CAAC,YAAY,IAAI,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,2BAA2B;QAC3B,kGAAkG;QAElG,gGAAgG;QAChG,MAAM,EAAE;aACL,eAAe,CACd;YACE,IAAI,EAAE,sBAAsB;SAC7B,EACD,EAAE,QAAQ,EAAE,KAAK,EAAE,CACpB;aACA,OAAO,EAAE,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAyB;IACvD,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE,GAAG,IAAI,CAAC,CAAC;AAC1D,CAAC"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -43,6 +43,7 @@ export interface NormalizedMongoConnectionConfig {
|
|
|
43
43
|
password?: string;
|
|
44
44
|
lookup?: LookupFunction;
|
|
45
45
|
postImages: PostImagesOption;
|
|
46
|
+
heartbeat_interval_seconds: number;
|
|
46
47
|
connectionParams: MongoConnectionParams;
|
|
47
48
|
}
|
|
48
49
|
export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
|
|
@@ -71,6 +72,10 @@ export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
|
|
|
71
72
|
reject_ip_ranges?: string[] | undefined;
|
|
72
73
|
}, string, t.CodecProps>, t.ObjectCodec<{
|
|
73
74
|
post_images: t.OptionalCodec<t.Codec<"off" | "auto_configure" | "read_only", "off" | "auto_configure" | "read_only", string, t.CodecProps>>;
|
|
75
|
+
/**
|
|
76
|
+
* Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
|
|
77
|
+
*/
|
|
78
|
+
heartbeat_interval_seconds: t.OptionalCodec<t.Codec<number | null, number | null, string, t.CodecProps>>;
|
|
74
79
|
}>>;
|
|
75
80
|
/**
|
|
76
81
|
* Config input specified when starting services
|
package/dist/types/types.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb/types';
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
3
|
import * as service_types from '@powersync/service-types';
|
|
3
4
|
import * as t from 'ts-codec';
|
|
4
5
|
export var PostImagesOption;
|
|
@@ -35,9 +36,16 @@ export var PostImagesOption;
|
|
|
35
36
|
*/
|
|
36
37
|
PostImagesOption["READ_ONLY"] = "read_only";
|
|
37
38
|
})(PostImagesOption || (PostImagesOption = {}));
|
|
39
|
+
const DEFAULT_PING_INTERVAL_SECONDS = 60;
|
|
40
|
+
const MIN_PING_INTERVAL_SECONDS = 5;
|
|
41
|
+
const MAX_PING_INTERVAL_SECONDS = 60;
|
|
38
42
|
export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(t.object({
|
|
39
43
|
// Replication specific settings
|
|
40
|
-
post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
|
|
44
|
+
post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional(),
|
|
45
|
+
/**
|
|
46
|
+
* Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
|
|
47
|
+
*/
|
|
48
|
+
heartbeat_interval_seconds: t.number.or(t.Null).optional()
|
|
41
49
|
}));
|
|
42
50
|
/**
|
|
43
51
|
* Validate and normalize connection options.
|
|
@@ -50,7 +58,17 @@ export function normalizeConnectionConfig(options) {
|
|
|
50
58
|
...base,
|
|
51
59
|
id: options.id ?? 'default',
|
|
52
60
|
tag: options.tag ?? 'default',
|
|
53
|
-
postImages: options.post_images ?? PostImagesOption.OFF
|
|
61
|
+
postImages: options.post_images ?? PostImagesOption.OFF,
|
|
62
|
+
heartbeat_interval_seconds: normalizeHeartbeatInterval(options.heartbeat_interval_seconds)
|
|
54
63
|
};
|
|
55
64
|
}
|
|
65
|
+
function normalizeHeartbeatInterval(value) {
|
|
66
|
+
if (value == null) {
|
|
67
|
+
return DEFAULT_PING_INTERVAL_SECONDS;
|
|
68
|
+
}
|
|
69
|
+
if (!Number.isFinite(value) || value < MIN_PING_INTERVAL_SECONDS || value > MAX_PING_INTERVAL_SECONDS) {
|
|
70
|
+
throw new ServiceError(ErrorCode.PSYNC_S1109, `MongoDB connection: heartbeat_interval_seconds must be between ${MIN_PING_INTERVAL_SECONDS} and ${MAX_PING_INTERVAL_SECONDS} seconds; null or omitted defaults to ${DEFAULT_PING_INTERVAL_SECONDS}`);
|
|
71
|
+
}
|
|
72
|
+
return value;
|
|
73
|
+
}
|
|
56
74
|
//# sourceMappingURL=types.js.map
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAClE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAED,MAAM,6BAA6B,GAAG,EAAE,CAAC;AACzC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AAqBrC,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC/G,CAAC,CAAC,MAAM,CAAC;IACP,gCAAgC;IAChC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnG;;OAEG;IACH,0BAA0B,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC3D,CAAC,CACH,CAAC;AAaF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAqC;IAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;QACzF,0BAA0B,EAAE,0BAA0B,CAAC,OAAO,CAAC,0BAA0B,CAAC;KAC3F,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,KAAgC;IAClE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,yBAAyB,IAAI,KAAK,GAAG,yBAAyB,EAAE,CAAC;QACtG,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,kEAAkE,yBAAyB,QAAQ,yBAAyB,yCAAyC,6BAA6B,EAAE,CACrM,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-mongodb",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.20.0",
|
|
5
5
|
"license": "FSL-1.1-ALv2",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
@@ -15,17 +15,17 @@
|
|
|
15
15
|
"bson": "^6.10.4",
|
|
16
16
|
"ts-codec": "^1.3.0",
|
|
17
17
|
"uuid": "^14.0.0",
|
|
18
|
-
"@powersync/lib-service-mongodb": "0.6.
|
|
19
|
-
"@powersync/lib-services-framework": "0.
|
|
20
|
-
"@powersync/service-core": "1.
|
|
18
|
+
"@powersync/lib-service-mongodb": "0.6.30",
|
|
19
|
+
"@powersync/lib-services-framework": "0.10.0",
|
|
20
|
+
"@powersync/service-core": "1.24.0",
|
|
21
21
|
"@powersync/service-jsonbig": "0.17.13",
|
|
22
|
-
"@powersync/service-sync-rules": "0.
|
|
23
|
-
"@powersync/service-types": "0.
|
|
22
|
+
"@powersync/service-sync-rules": "0.40.0",
|
|
23
|
+
"@powersync/service-types": "0.17.0"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
|
-
"@powersync/service-core-tests": "0.
|
|
27
|
-
"@powersync/service-module-mongodb-storage": "0.
|
|
28
|
-
"@powersync/service-module-postgres-storage": "0.
|
|
26
|
+
"@powersync/service-core-tests": "0.18.0",
|
|
27
|
+
"@powersync/service-module-mongodb-storage": "0.19.0",
|
|
28
|
+
"@powersync/service-module-postgres-storage": "0.17.0"
|
|
29
29
|
},
|
|
30
30
|
"scripts": {
|
|
31
31
|
"build": "tsc -b",
|
|
@@ -4,11 +4,12 @@ import { api, ParseSyncConfigOptions, ReplicationHeadCallback } from '@powersync
|
|
|
4
4
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
5
5
|
import * as service_types from '@powersync/service-types';
|
|
6
6
|
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
8
|
+
import { CheckpointImplementation } from '../replication/checkpoints/CheckpointImplementation.js';
|
|
9
|
+
import { createCheckpointImplementation } from '../replication/checkpoints/create-checkpoint-implementation.js';
|
|
9
10
|
import { MongoManager } from '../replication/MongoManager.js';
|
|
10
|
-
import { constructAfterRecord
|
|
11
|
-
import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js';
|
|
11
|
+
import { constructAfterRecord } from '../replication/MongoRelation.js';
|
|
12
|
+
import { CHECKPOINTS_COLLECTION, detectDocumentDb } from '../replication/replication-utils.js';
|
|
12
13
|
import * as types from '../types/types.js';
|
|
13
14
|
import { escapeRegExp } from '../utils.js';
|
|
14
15
|
|
|
@@ -19,6 +20,8 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
19
20
|
connectionTag: string;
|
|
20
21
|
defaultSchema: string;
|
|
21
22
|
|
|
23
|
+
private checkpointImplementation: CheckpointImplementation | null = null;
|
|
24
|
+
|
|
22
25
|
constructor(protected config: types.ResolvedConnectionConfig) {
|
|
23
26
|
const manager = new MongoManager(config);
|
|
24
27
|
this.client = manager.client;
|
|
@@ -199,41 +202,27 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
199
202
|
}
|
|
200
203
|
|
|
201
204
|
async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
|
|
202
|
-
const
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
const head = session.clusterTime?.clusterTime;
|
|
206
|
-
if (head == null) {
|
|
207
|
-
throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
const r = await callback(new MongoLSN({ timestamp: head }).comparable);
|
|
211
|
-
|
|
212
|
-
// Trigger a change on the changestream.
|
|
213
|
-
await this.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
|
|
214
|
-
{
|
|
215
|
-
_id: STANDALONE_CHECKPOINT_ID as any
|
|
216
|
-
},
|
|
217
|
-
{
|
|
218
|
-
$inc: { i: 1 }
|
|
219
|
-
},
|
|
220
|
-
{
|
|
221
|
-
upsert: true,
|
|
222
|
-
returnDocument: 'after',
|
|
223
|
-
session
|
|
224
|
-
}
|
|
225
|
-
);
|
|
226
|
-
const time = session.operationTime!;
|
|
227
|
-
if (time == null) {
|
|
228
|
-
throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
|
|
229
|
-
} else if (time.lt(head)) {
|
|
230
|
-
throw new ServiceAssertionError(`operationTime must be > clusterTime`);
|
|
231
|
-
}
|
|
205
|
+
const checkpointImplementation = await this.getCheckpointImplementation();
|
|
206
|
+
return checkpointImplementation.createReplicationHead(callback);
|
|
207
|
+
}
|
|
232
208
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
await
|
|
209
|
+
private async getCheckpointImplementation(): Promise<CheckpointImplementation> {
|
|
210
|
+
if (this.checkpointImplementation == null) {
|
|
211
|
+
const isDocumentDb = await detectDocumentDb(this.db);
|
|
212
|
+
this.checkpointImplementation = createCheckpointImplementation(isDocumentDb, {
|
|
213
|
+
client: this.client,
|
|
214
|
+
db: this.db,
|
|
215
|
+
// The adapter never streams, so it has no real barrier document. This
|
|
216
|
+
// random id is only safe because the adapter only ever calls
|
|
217
|
+
// createReplicationHead, which produces a standalone (stream_id = null)
|
|
218
|
+
// head that every real ChangeStream observes. It must NOT be used for a
|
|
219
|
+
// batch barrier (createBatchCheckpoint stamps this id), or the head would
|
|
220
|
+
// become an own-barrier of a phantom stream that nothing resolves.
|
|
221
|
+
checkpointStreamId: new mongo.ObjectId(),
|
|
222
|
+
logger
|
|
223
|
+
});
|
|
236
224
|
}
|
|
225
|
+
return this.checkpointImplementation;
|
|
237
226
|
}
|
|
238
227
|
|
|
239
228
|
async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { storage } from '@powersync/service-core';
|
|
3
|
+
|
|
4
|
+
export type SentinelLSNSpecification = {
|
|
5
|
+
sentinel: bigint;
|
|
6
|
+
/**
|
|
7
|
+
* Resume tokens are opaque on sentinel-based sources (e.g. DocumentDB). The
|
|
8
|
+
* sentinel component is the comparable position; this token is only used to
|
|
9
|
+
* resume the change stream.
|
|
10
|
+
*/
|
|
11
|
+
resume_token?: mongo.ResumeToken | null;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const DELIMINATOR = '|';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Width of the sentinel coordinate, matching a MongoDB cluster timestamp
|
|
18
|
+
* (64-bit → 16 hex chars). The sentinel counter is a 64-bit `Long`, so this is
|
|
19
|
+
* sufficient and never truncates.
|
|
20
|
+
*/
|
|
21
|
+
const SENTINEL_HEX_LENGTH = 16;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* LSN for sentinel-based checkpointing (sources without a usable clusterTime,
|
|
25
|
+
* e.g. DocumentDB). The ordered coordinate is a monotonic sentinel counter; the
|
|
26
|
+
* opaque resume token is carried alongside it only for `resumeAfter`.
|
|
27
|
+
*
|
|
28
|
+
* The coordinate is serialized as a 16-hex-char value with the **same shape as
|
|
29
|
+
* a MongoDB timestamp LSN** ({@link MongoLSN}): the high 32 bits resemble epoch
|
|
30
|
+
* seconds and the low 32 bits an increment. This is deliberate — it makes
|
|
31
|
+
* sentinel LSNs directly string-comparable with timestamp LSNs, so a sentinel
|
|
32
|
+
* coordinate (seeded at the current epoch seconds; see createSentinelCheckpointLsn)
|
|
33
|
+
* always sorts **above** any real-timestamp LSN issued in the past.
|
|
34
|
+
*
|
|
35
|
+
* It only *resembles* a timestamp. The value is a synthetic monotonic counter,
|
|
36
|
+
* not a real cluster time, and must never be used as one (e.g. it is never fed
|
|
37
|
+
* to `startAtOperationTime`; the sentinel implementation resumes purely via the
|
|
38
|
+
* resume token).
|
|
39
|
+
*/
|
|
40
|
+
export class SentinelLSN {
|
|
41
|
+
static ZERO = new SentinelLSN({ sentinel: 0n });
|
|
42
|
+
|
|
43
|
+
static fromSerialized(comparable: string): SentinelLSN {
|
|
44
|
+
const [sentinelString, resumeString] = comparable.split(DELIMINATOR);
|
|
45
|
+
|
|
46
|
+
return new SentinelLSN({
|
|
47
|
+
sentinel: BigInt(`0x${sentinelString}`),
|
|
48
|
+
resume_token: resumeString ? storage.deserializeBson(Buffer.from(resumeString, 'base64')).resumeToken : null
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
constructor(protected options: SentinelLSNSpecification) {}
|
|
53
|
+
|
|
54
|
+
get sentinel() {
|
|
55
|
+
return this.options.sentinel;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
get resumeToken() {
|
|
59
|
+
return this.options.resume_token;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
get comparable() {
|
|
63
|
+
// padStart(16) of the combined 64-bit value yields the identical string to
|
|
64
|
+
// MongoLSN's high(8)+low(8) formatting, keeping the two formats comparable.
|
|
65
|
+
const sentinel = this.sentinel.toString(16).padStart(SENTINEL_HEX_LENGTH, '0');
|
|
66
|
+
const segments = [sentinel];
|
|
67
|
+
|
|
68
|
+
if (this.resumeToken) {
|
|
69
|
+
segments.push(storage.serializeBson({ resumeToken: this.resumeToken }).toString('base64'));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return segments.join(DELIMINATOR);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
toString() {
|
|
76
|
+
return this.comparable;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
@@ -39,7 +39,8 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
|
|
|
39
39
|
storageEngine: context.storageEngine,
|
|
40
40
|
metricsEngine: context.metricsEngine,
|
|
41
41
|
connectionFactory: connectionFactory,
|
|
42
|
-
rateLimiter: new MongoErrorRateLimiter()
|
|
42
|
+
rateLimiter: new MongoErrorRateLimiter(),
|
|
43
|
+
heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
|
|
43
44
|
});
|
|
44
45
|
}
|
|
45
46
|
|