@powersync/service-module-mongodb 0.0.0-dev-20260803144812 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -7
- package/dist/module/MongoModule.js +2 -1
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +8 -0
- package/dist/replication/ChangeStream.js +71 -14
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +2 -1
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/MongoSnapshotter.js +6 -6
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +3 -11
- package/dist/replication/RawChangeStream.js +0 -17
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/checkpoints/CheckpointImplementation.d.ts +4 -1
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.d.ts +4 -1
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js +1 -1
- package/dist/replication/checkpoints/SentinelCheckpointImplementation.js.map +1 -1
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.d.ts +4 -1
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js +2 -1
- package/dist/replication/checkpoints/TimestampCheckpointImplementation.js.map +1 -1
- package/dist/types/types.d.ts +5 -0
- package/dist/types/types.js +20 -2
- package/dist/types/types.js.map +1 -1
- package/package.json +9 -9
- package/src/module/MongoModule.ts +2 -1
- package/src/replication/ChangeStream.ts +75 -16
- package/src/replication/ChangeStreamReplicationJob.ts +2 -1
- package/src/replication/MongoSnapshotter.ts +6 -6
- package/src/replication/RawChangeStream.ts +3 -30
- package/src/replication/checkpoints/CheckpointImplementation.ts +1 -1
- package/src/replication/checkpoints/SentinelCheckpointImplementation.ts +2 -2
- package/src/replication/checkpoints/TimestampCheckpointImplementation.ts +3 -2
- package/src/types/types.ts +27 -2
- package/test/src/change_stream.test.ts +73 -0
- package/test/src/config.test.ts +34 -0
- package/test/src/documentdb_mode.test.ts +1 -2
- package/test/src/raw_change_stream.test.ts +0 -63
- package/test/src/util.ts +11 -29
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -7,19 +7,11 @@ export interface RawChangeStreamOptions {
|
|
|
7
7
|
* How long to wait for new data per batch (max time for long-polling).
|
|
8
8
|
* This is sent as maxTimeMS for the getMore command.
|
|
9
9
|
*
|
|
10
|
-
* A value of 0
|
|
11
|
-
*
|
|
10
|
+
* A value of 0 removes the explicit await limit and leaves the server's
|
|
11
|
+
* default awaitData behavior in effect; it does not make getMore return
|
|
12
|
+
* immediately. Snapshot probes use 0 for this purpose.
|
|
12
13
|
*/
|
|
13
14
|
maxAwaitTimeMS: number;
|
|
14
|
-
/**
|
|
15
|
-
* Also enforce maxAwaitTimeMS on the client for empty getMore batches.
|
|
16
|
-
*
|
|
17
|
-
* Azure DocumentDB currently returns idle getMore calls before maxTimeMS. When
|
|
18
|
-
* this is enabled, empty batches are delayed locally (capped at 1s) to avoid
|
|
19
|
-
* tight polling. We still send maxTimeMS so this remains compatible with
|
|
20
|
-
* servers that handle maxAwaitTimeMS correctly.
|
|
21
|
-
*/
|
|
22
|
-
clientSideMaxAwaitTimeMS?: boolean;
|
|
23
15
|
/**
|
|
24
16
|
* Timeout for the initial aggregate command.
|
|
25
17
|
*/
|
|
@@ -1,11 +1,6 @@
|
|
|
1
1
|
import { isMaxTimeMSExpiredError, isMongoServerError, isTimeoutError, mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { DatabaseConnectionError, ErrorCode, ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
-
import { performance } from 'node:perf_hooks';
|
|
4
|
-
import { setTimeout as delay } from 'node:timers/promises';
|
|
5
3
|
import { ChangeStreamInvalidatedError } from './ChangeStream.js';
|
|
6
|
-
// Keep the DocumentDB idle-poll workaround from adding the full maxAwaitTimeMS
|
|
7
|
-
// as local latency when an update arrives just after an empty batch.
|
|
8
|
-
const CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS = 1_000;
|
|
9
4
|
const deserialize = mongo.BSON.deserialize;
|
|
10
5
|
const DESERIALIZE_DEFAULT = { useBigInt64: true };
|
|
11
6
|
const DESERIALIZE_RAW = {
|
|
@@ -135,17 +130,12 @@ async function* rawChangeStreamInner(session, db, pipeline, options, batchSizer)
|
|
|
135
130
|
while (cursorId && cursorId !== 0n) {
|
|
136
131
|
options.signal?.throwIfAborted();
|
|
137
132
|
using commandSpan = options.tracer?.span('changestream', 'getmore');
|
|
138
|
-
const getMoreStartedAt = performance.now();
|
|
139
133
|
const getMoreCommand = {
|
|
140
134
|
getMore: cursorId,
|
|
141
135
|
collection: nsCollection,
|
|
142
136
|
batchSize: batchSizer.next(),
|
|
143
137
|
maxTimeMS: options.maxAwaitTimeMS
|
|
144
138
|
};
|
|
145
|
-
// Azure DocumentDB currently returns empty getMore batches before
|
|
146
|
-
// maxTimeMS expires. Keep maxTimeMS for forward compatibility with the
|
|
147
|
-
// server-side behavior, and when client-side mode is enabled, enforce the
|
|
148
|
-
// capped idle wait locally for empty batches below.
|
|
149
139
|
const getMoreResult = await db.command(getMoreCommand, { session, raw: true }).catch((e) => {
|
|
150
140
|
if (isMongoServerError(e) && e.codeName == 'CursorKilled') {
|
|
151
141
|
// This may be due to the killCursors command issued when aborting.
|
|
@@ -169,13 +159,6 @@ async function* rawChangeStreamInner(session, db, pipeline, options, batchSizer)
|
|
|
169
159
|
// postBatchResumeToken is returned in MongoDB 4.0.7 and later, and we support 6.0+
|
|
170
160
|
throw new ReplicationAssertionError(`postBatchResumeToken from aggregate response`);
|
|
171
161
|
}
|
|
172
|
-
if (options.clientSideMaxAwaitTimeMS && nextBatch.length == 0) {
|
|
173
|
-
const remainingMaxAwaitTimeMS = Math.ceil(options.maxAwaitTimeMS - (performance.now() - getMoreStartedAt));
|
|
174
|
-
if (remainingMaxAwaitTimeMS > 0) {
|
|
175
|
-
const clientSideDelayMs = Math.min(remainingMaxAwaitTimeMS, CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS);
|
|
176
|
-
await delay(clientSideDelayMs, undefined, { signal: options.signal });
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
162
|
yield {
|
|
180
163
|
events: nextBatch,
|
|
181
164
|
resumeToken: cursor.postBatchResumeToken,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RawChangeStream.js","sourceRoot":"","sources":["../../src/replication/RawChangeStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACpH,OAAO,EACL,uBAAuB,EACvB,SAAS,EAET,yBAAyB,EAC1B,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"RawChangeStream.js","sourceRoot":"","sources":["../../src/replication/RawChangeStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACpH,OAAO,EACL,uBAAuB,EACvB,SAAS,EAET,yBAAyB,EAC1B,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAgDjE,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;AAC3C,MAAM,mBAAmB,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AAClD,MAAM,eAAe,GAAkC;IACrD,GAAG,mBAAmB;IACtB,GAAG,EAAE,IAAI;IACT,6DAA6D;IAC7D,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;CAC5B,CAAC;AACF,MAAM,yBAAyB,GAAG;IAChC,GAAG,mBAAmB;IACtB,kEAAkE;IAClE,WAAW,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAClD,+DAA+D;IAC/D,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;CAC5B,CAAC;AAEF,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,EAAY,EAAE,QAA0B,EAAE,OAA+B;IAC9G,4IAA4I;IAC5I,2BAA2B;IAC3B,gDAAgD;IAChD,wDAAwD;IACxD,sGAAsG;IACtG,IAAI,CAAC,CAAC,eAAe,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,yBAAyB,CAAC,4CAA4C,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,eAAe,GAAmB,IAAI,CAAC;IAE3C,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAE5D,qHAAqH;IACrH,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACzC,YAAY,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;IAEtE,OAAO,IAAI,EAAE,CAAC;QACZ,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,IAAI,aAAa,GAAG,QAAQ,CAAC;YAC7B,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC;gBAClC,MAAM,iBAAiB,GAAG,EAAE,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;gBACrD,OAAO,iBAAiB,CAAC,oBAAoB,CAAC;gBAC9C,OAAO,iBAAiB,CAAC,UAAU,CAAC;gBACpC,iBAAiB,CAAC,WAAW,GAAG,eAAe,CAAC;gBAChD,aAAa,GAAG,CAAC,EAAE,aAAa,EAAE,iBAAiB,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;YAClE,CAAC;YACD,MAAM,KAAK,GAAG,oBAAoB,CAChC,OAAO,EACP,EAAE,EACF,aAAa,EACb;gBACE,GAAG,OAAO;aACX,EACD,UAAU,CACX,CAAC;YACF,IAAI,KAAK,EAAE,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC9B,MAAM,KAAK,CAAC;gBACZ,eAAe,GAAG,KAAK,CAAC,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,0BAA0B,EAAE,CAAC;gBAC5C,iDAAiD;gBACjD,0FAA0F;gBAC1F,uDAAuD;gBACvD,0CAA0C;gBAC1C,0LAA0L;gBAC1L,oIAAoI;gBACpI,+GAA+G;gBAE/G,4GAA4G;gBAC5G,6EAA6E;gBAC7E,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,4CAA4C,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACvF,SAAS;YACX,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,oBAAoB,CAClC,OAA4B,EAC5B,EAAY,EACZ,QAA0B,EAC1B,OAA+B,EAC/B,UAA6B;IAE7B,aAAa;IACb,gGAAgG;IAEhG,IAAI,QAAQ,GAAkB,IAAI,CAAC;IAEnC;;OAEG;IACH,IAAI,YAAY,GAAkB,IAAI,CAAC;IAEvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;IAC3C,IAAI,YAAY,GAAwB,IAAI,CAAC;IAE7C,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,KAAK,EAAE,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YAChE,6CAA6C;YAC7C,YAAY,GAAG,EAAE;iBACd,OAAO,CAAC;gBACP,WAAW,EAAE,YAAY;gBACzB,OAAO,EAAE,CAAC,QAAQ,CAAC;aACpB,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IACF,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAEnD,IAAI,CAAC;QACH,CAAC;YACC,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;YACxE,gEAAgE;YAChE,MAAM,eAAe,GAAG,MAAM,EAAE;iBAC7B,OAAO,CACN;gBACE,SAAS,EAAE,UAAU;gBACrB,QAAQ;gBACR,MAAM,EAAE;oBACN,+DAA+D;oBAC/D,yDAAyD;oBACzD,SAAS,EAAE,CAAC;iBACb;gBACD,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,EACD,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CACvB;iBACA,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YAEL,aAAa,EAAE,GAAG,EAAE,CAAC;YAErB,MAAM,MAAM,GAAG,WAAW,CAAC,eAAe,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;YAE9E,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC7B,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAE9C,IAAI,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC;YAE9B,IAAI,MAAM,CAAC,oBAAoB,IAAI,IAAI,EAAE,CAAC;gBACxC,yFAAyF;gBACzF,mFAAmF;gBACnF,MAAM,IAAI,yBAAyB,CAAC,8CAA8C,CAAC,CAAC;YACtF,CAAC;YAED,MAAM;gBACJ,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,MAAM,CAAC,oBAAoB;gBACxC,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU;aAC5C,CAAC;QACJ,CAAC;QAED,wDAAwD;QACxD,OAAO,QAAQ,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;YACnC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAEjC,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YACpE,MAAM,cAAc,GAAmB;gBACrC,OAAO,EAAE,QAAQ;gBACjB,UAAU,EAAE,YAAY;gBACxB,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE;gBAC5B,SAAS,EAAE,OAAO,CAAC,cAAc;aAClC,CAAC;YACF,MAAM,aAAa,GAAmB,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACzG,IAAI,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,EAAE,CAAC;oBAC1D,mEAAmE;oBACnE,6CAA6C;oBAC7C,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;gBACnC,CAAC;gBAED,IAAI,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,IAAI,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;wBACtB,UAAU,CAAC,gBAAgB,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,IAAI,0BAA0B,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBAChE,CAAC;gBACD,MAAM,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,WAAW,EAAE,GAAG,EAAE,CAAC;YAEnB,MAAM,MAAM,GAAG,WAAW,CAAC,aAAa,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;YAC5E,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAEnC,IAAI,MAAM,CAAC,oBAAoB,IAAI,IAAI,EAAE,CAAC;gBACxC,yFAAyF;gBACzF,mFAAmF;gBACnF,MAAM,IAAI,yBAAyB,CAAC,8CAA8C,CAAC,CAAC;YACtF,CAAC;YACD,MAAM;gBACJ,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,MAAM,CAAC,oBAAoB;gBACxC,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,UAAU;aAC1C,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,IAAI,yBAAyB,CAAC,kCAAkC,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACtD,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,0EAA0E;YAC1E,MAAM,YAAY,CAAC;QACrB,CAAC;aAAM,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,KAAK,EAAE,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACvE,MAAM,EAAE;iBACL,OAAO,CAAC;gBACP,WAAW,EAAE,YAAY;gBACzB,OAAO,EAAE,CAAC,QAAQ,CAAC;aACpB,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,0BAA2B,SAAQ,KAAK;CAAG;AAEjD;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEhC;;;;;;;;;;;;;;;GAeG;AACH,MAAM,iBAAiB;IAGD;IAFZ,aAAa,CAAS;IAE9B,YAAoB,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QACtC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,IAAI;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,aAAa,GAAG,qBAAqB,CAAC,CAAC;QAC7F,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,IAAI,CAAC,aAAa,GAAG,qBAAqB,CAAC;IAC7C,CAAC;CACF;AAiDD;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAChD,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,CAA8B,CAAC;IAC9E,kCAAkC;IAClC,GAAG,CAAC,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,mBAAmB,CAAQ,CAAC;IAC3D,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;QACrB,GAAG,CAAC,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,mBAAmB,CAAQ,CAAC;IAC/D,CAAC;IACD,IAAI,GAAG,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QAC3B,GAAG,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,UAAU,EAAE,mBAAmB,CAAQ,CAAC;IAC3E,CAAC;IAED,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;QAChB,GAAG,CAAC,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,mBAAmB,CAAQ,CAAC;IAC3D,CAAC;IACD,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;QAChB,GAAG,CAAC,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,mBAAmB,CAAQ,CAAC;IAC3D,CAAC;IACD,IAAI,aAAa,IAAI,GAAG,EAAE,CAAC;QACzB,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,mBAAmB,CAAQ,CAAC;IAC7E,CAAC;IACD,OAAO,GAAU,CAAC;AACpB,CAAC;AAED,SAAS,4BAA4B,CAAC,CAAU;IAC9C,qHAAqH;IACrH,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,mGAAmG;QACnG,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QAC1C,+CAA+C;QAC/C,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,CAAC,CAAC,aAAa,CAAC,4BAA4B,CAAC,EAAE,CAAC;QACzD,8IAA8I;QAC9I,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,uBAAuB,CAAC,CAAC,CAAC,EAAE,CAAC;QACtC,4CAA4C;QAC5C,wIAAwI;QACxI,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,CAAC;QACN,gCAAgC;QAChC,oGAAoG;QACpG,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAU;IAC7C,IAAI,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;QACtB,oCAAoC;QACpC,iGAAiG;QACjG,8CAA8C;QAC9C,gCAAgC;QAChC,6JAA6J;QAC7J,MAAM,IAAI,uBAAuB,CAAC,SAAS,CAAC,WAAW,EAAE,4CAA4C,EAAE,CAAC,CAAC,CAAC;IAC5G,CAAC;SAAM,IACL,kBAAkB,CAAC,CAAC,CAAC;QACrB,CAAC,CAAC,QAAQ,IAAI,oBAAoB;QAClC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,0BAA0B,CAAC,EAC9C,CAAC;QACD,MAAM,IAAI,4BAA4B,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;SAAM,IAAI,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,+BAA+B,CAAC,EAAE,CAAC;QACrF,MAAM,IAAI,4BAA4B,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,uBAAuB,CAAC,SAAS,CAAC,WAAW,EAAE,oCAAoC,EAAE,CAAC,CAAC,CAAC;IACpG,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,EAAU;IAC5C,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,EAAE,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -112,7 +112,10 @@ export interface CheckpointImplementation {
|
|
|
112
112
|
* The timestamp implementation parses the timestamp embedded in the token.
|
|
113
113
|
* The sentinel implementation pairs the token with the current coordinate.
|
|
114
114
|
*/
|
|
115
|
-
lsnFromResumeToken(resumeToken: mongo.ResumeToken):
|
|
115
|
+
lsnFromResumeToken(resumeToken: mongo.ResumeToken): {
|
|
116
|
+
lsn: string;
|
|
117
|
+
timestamp: Date | null;
|
|
118
|
+
};
|
|
116
119
|
/**
|
|
117
120
|
* Source-side replication head for write checkpoints. The LSN passed to the
|
|
118
121
|
* callback must compare at or below any LSN committed after the caller's
|
|
@@ -36,7 +36,10 @@ export declare class SentinelCheckpointImplementation implements CheckpointImple
|
|
|
36
36
|
createBatchCheckpoint(): Promise<string>;
|
|
37
37
|
createFirstBarrier(): Promise<string | null>;
|
|
38
38
|
keepalive(_batch: storage.BucketStorageBatch, _resumeToken: mongo.ResumeToken): Promise<void>;
|
|
39
|
-
lsnFromResumeToken(resumeToken: mongo.ResumeToken):
|
|
39
|
+
lsnFromResumeToken(resumeToken: mongo.ResumeToken): {
|
|
40
|
+
lsn: string;
|
|
41
|
+
timestamp: null;
|
|
42
|
+
};
|
|
40
43
|
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
41
44
|
readonly event: CheckpointEventApi;
|
|
42
45
|
readonly checkpointClearFilter: mongo.Filter<mongo.Document>;
|
|
@@ -93,7 +93,7 @@ export class SentinelCheckpointImplementation {
|
|
|
93
93
|
// Pair the bare token with the current coordinate. The coordinate is
|
|
94
94
|
// frozen between checkpoint events, so a per-batch resume marker only
|
|
95
95
|
// advances the token; that is all resumption needs (see the design doc).
|
|
96
|
-
return new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable;
|
|
96
|
+
return { lsn: new SentinelLSN({ sentinel: this.position, resume_token: resumeToken }).comparable, timestamp: null };
|
|
97
97
|
}
|
|
98
98
|
async createReplicationHead(callback) {
|
|
99
99
|
const head = await createSentinelCheckpointLsn(this.context.client, this.context.db);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SentinelCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/SentinelCheckpointImplementation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EAKL,eAAe,EAEhB,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,gCAAgC;IAWvB;IAVX,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;IAE/C;;;;;OAKG;IACK,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;IAE7C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACvE,CAAC;IAED,YAAY,CAAC,GAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrG,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,sEAAsE;QACtE,+DAA+D;QAC/D,6CAA6C;QAC7C,6EAA6E;QAC7E,4EAA4E;QAC5E,qCAAqC;QACrC,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5G,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,uEAAuE;QACvE,2EAA2E;QAC3E,0EAA0E;QAC1E,mBAAmB;QACnB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAkC,EAAE,YAA+B;QACjF,yEAAyE;QACzE,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,kDAAkD;QAClD,EAAE;QACF,0EAA0E;QAC1E,0EAA0E;QAC1E,sEAAsE;QACtE,sEAAsE;QACtE,iFAAiF;QACjF,0EAA0E;QAC1E,yDAAyD;QACzD,EAAE;QACF,yEAAyE;QACzE,yEAAyE;QACzE,qEAAqE;QACrE,sEAAsE;QACtE,kEAAkE;QAClE,2DAA2D;QAC3D,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACzG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,OAAO,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"SentinelCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/SentinelCheckpointImplementation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EAKL,eAAe,EAEhB,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,gCAAgC;IAWvB;IAVX,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;IAE/C;;;;;OAKG;IACK,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;IAE7C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACvE,CAAC;IAED,YAAY,CAAC,GAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACrG,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,sEAAsE;QACtE,+DAA+D;QAC/D,6CAA6C;QAC7C,6EAA6E;QAC7E,4EAA4E;QAC5E,qCAAqC;QACrC,OAAO,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5G,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,uEAAuE;QACvE,2EAA2E;QAC3E,0EAA0E;QAC1E,mBAAmB;QACnB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAkC,EAAE,YAA+B;QACjF,yEAAyE;QACzE,uEAAuE;QACvE,qEAAqE;QACrE,4EAA4E;QAC5E,kDAAkD;QAClD,EAAE;QACF,0EAA0E;QAC1E,0EAA0E;QAC1E,sEAAsE;QACtE,sEAAsE;QACtE,iFAAiF;QACjF,0EAA0E;QAC1E,yDAAyD;QACzD,EAAE;QACF,yEAAyE;QACzE,yEAAyE;QACzE,qEAAqE;QACrE,sEAAsE;QACtE,kEAAkE;QAClE,2DAA2D;QAC3D,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACzG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,OAAO,EAAE,GAAG,EAAE,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACtH,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,IAAI,GAAG,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACrF,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QACzD,IAAI,aAAa,EAAE,CAAC;YAClB,uEAAuE;YACvE,qEAAqE;YACrE,uCAAuC;YACvC,8EAA8E;YAC9E,MAAM,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBAC3C,mEAAmE;gBACnE,qEAAqE;gBACrE,wBAAwB;gBACxB,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,iEAAiE;gBACjE,uEAAuE;gBACvE,wEAAwE;gBACxE,qDAAqD;gBACrD,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;YAEnC,IAAI,IAAyB,CAAC;YAC9B,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,mEAAmE;gBACnE,+DAA+D;gBAC/D,IAAI,GAAG,YAAY,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5D,wEAAwE;gBACxE,IAAI,GAAG,aAAa,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,6CAA6C;gBAC7C,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,4EAA4E;YAC5E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E,EAC1E,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,aAAa,4CAA4C,CAAC,CACvF,CAAC;YACJ,CAAC;YAED,wEAAwE;YACxE,qEAAqE;YACrE,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,IAAI,WAAW,CAAC;gBACrB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;gBACpC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,MAAM,OAAO,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;gBAClG,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC;QACnG,CAAC;KACF,CAAC;IAEF,wEAAwE;IACxE,4EAA4E;IAC5E,8EAA8E;IAC9E,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAC5E,uEAAuE;IACvE,2EAA2E;IAC3E,sCAAsC;IACtC,EAAE;IACF,uEAAuE;IACvE,yEAAyE;IACzE,yEAAyE;IACzE,2EAA2E;IAC3E,sEAAsE;IACtE,yEAAyE;IACzE,sEAAsE;IACtE,gCAAgC;IACvB,qBAAqB,GAAiC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,sBAAsB,EAAS,EAAE,CAAC;IAE/G,wGAAwG;IAChG,aAAa,CAAC,QAAgB;QACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,YAAY,CAAC,OAAuB;QAC1C,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,4BAA4B,CACpC,0EAA0E;YAC1E,4EAA4E;YAC5E,IAAI,KAAK,CAAC,iCAAiC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CACzE,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC;IACnB,CAAC;CACF;AAYD,SAAS,uBAAuB,CAAC,GAAkC;IACjE,MAAM,IAAI,GAAG,GAAmC,CAAC;IACjD,IAAI,IAAI,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,YAAsB,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -17,7 +17,10 @@ export declare class TimestampCheckpointImplementation implements CheckpointImpl
|
|
|
17
17
|
createBatchCheckpoint(): Promise<string>;
|
|
18
18
|
createFirstBarrier(): Promise<string | null>;
|
|
19
19
|
keepalive(batch: storage.BucketStorageBatch, resumeToken: mongo.ResumeToken): Promise<void>;
|
|
20
|
-
lsnFromResumeToken(resumeToken: mongo.ResumeToken):
|
|
20
|
+
lsnFromResumeToken(resumeToken: mongo.ResumeToken): {
|
|
21
|
+
lsn: string;
|
|
22
|
+
timestamp: Date;
|
|
23
|
+
};
|
|
21
24
|
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
22
25
|
readonly event: CheckpointEventApi;
|
|
23
26
|
readonly checkpointClearFilter: mongo.Filter<mongo.Document>;
|
|
@@ -49,7 +49,8 @@ export class TimestampCheckpointImplementation {
|
|
|
49
49
|
}
|
|
50
50
|
lsnFromResumeToken(resumeToken) {
|
|
51
51
|
// The timestamp is embedded in the resume token.
|
|
52
|
-
|
|
52
|
+
const lsn = MongoLSN.fromResumeToken(resumeToken);
|
|
53
|
+
return { lsn: lsn.comparable, timestamp: timestampToDate(lsn.timestamp) };
|
|
53
54
|
}
|
|
54
55
|
async createReplicationHead(callback) {
|
|
55
56
|
const session = this.context.client.startSession();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TimestampCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/TimestampCheckpointImplementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACzG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAIL,eAAe,EACf,iBAAiB,EAElB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,iCAAiC;IAGxB;IAFX,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;IAE5C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;IACnF,CAAC;IAED,YAAY,CAAC,IAAmB;QAC9B,wEAAwE;IAC1E,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,6EAA6E;QAC7E,0BAA0B;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACtG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,uBAAuB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,MAAM,iBAAiB,eAAe,GAAG,CACjG,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,8EAA8E;QAC9E,OAAO,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiC,EAAE,WAA8B;QAC/E,oEAAoE;QACpE,kEAAkE;QAClE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,mGAAmG;QACnG,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,iDAAiD,eAAe,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,iDAAiD;QACjD,
|
|
1
|
+
{"version":3,"file":"TimestampCheckpointImplementation.js","sourceRoot":"","sources":["../../../src/replication/checkpoints/TimestampCheckpointImplementation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACzG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAIL,eAAe,EACf,iBAAiB,EAElB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,iCAAiC;IAGxB;IAFX,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;IAE5C,YAAoB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;IAAG,CAAC;IAEhE,mBAAmB,CAAC,GAAW;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;IACnF,CAAC;IAED,YAAY,CAAC,IAAmB;QAC9B,wEAAwE;IAC1E,CAAC;IAED,SAAS,CAAC,GAAW;QACnB,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,6EAA6E;QAC7E,0BAA0B;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACtG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,uBAAuB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,MAAM,iBAAiB,eAAe,GAAG,CACjG,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,0BAA0B;QAC9B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,8EAA8E;QAC9E,OAAO,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiC,EAAE,WAA8B;QAC/E,oEAAoE;QACpE,kEAAkE;QAClE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,mGAAmG;QACnG,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CACtB,iDAAiD,eAAe,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,WAA8B;QAC/C,iDAAiD;QACjD,MAAM,GAAG,GAAG,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAClD,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC;YAC9C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;YACpF,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAEjG,IAAI,aAAa,EAAE,CAAC;gBAClB,qEAAqE;gBACrE,iDAAiD;gBACjD,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,gBAAgB,CACvE;oBACE,GAAG,EAAE,wBAA+B;iBACrC,EACD;oBACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;iBACf,EACD;oBACE,MAAM,EAAE,IAAI;oBACZ,cAAc,EAAE,OAAO;oBACvB,OAAO;iBACR,CACF,CAAC;gBACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;gBACpC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,MAAM,IAAI,qBAAqB,CAAC,kDAAkD,CAAC,CAAC;gBACtF,CAAC;qBAAM,IAAI,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAEQ,KAAK,GAAuB;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,IAAI,sBAAsB,EAAE,CAAC;gBACnE,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,sFAAsF;YACtF,IAAI,YAAY,IAAI,wBAAwB,EAAE,CAAC;gBAC7C,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1F,CAAC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE;YACX,OAAO,IAAI,QAAQ,CAAC;gBAClB,SAAS,EAAE,iBAAiB,CAAC,GAAG,CAAC;gBACjC,YAAY,EAAE,GAAG,CAAC,GAAG;aACtB,CAAC,CAAC,UAAU,CAAC;QAChB,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YAC/B,8DAA8D;YAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;QACvC,CAAC;KACF,CAAC;IAEF,gFAAgF;IACvE,qBAAqB,GAAiC,EAAE,CAAC;CACnE"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -43,6 +43,7 @@ export interface NormalizedMongoConnectionConfig {
|
|
|
43
43
|
password?: string;
|
|
44
44
|
lookup?: LookupFunction;
|
|
45
45
|
postImages: PostImagesOption;
|
|
46
|
+
heartbeat_interval_seconds: number;
|
|
46
47
|
connectionParams: MongoConnectionParams;
|
|
47
48
|
}
|
|
48
49
|
export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
|
|
@@ -71,6 +72,10 @@ export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
|
|
|
71
72
|
reject_ip_ranges?: string[] | undefined;
|
|
72
73
|
}, string, t.CodecProps>, t.ObjectCodec<{
|
|
73
74
|
post_images: t.OptionalCodec<t.Codec<"off" | "auto_configure" | "read_only", "off" | "auto_configure" | "read_only", string, t.CodecProps>>;
|
|
75
|
+
/**
|
|
76
|
+
* Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
|
|
77
|
+
*/
|
|
78
|
+
heartbeat_interval_seconds: t.OptionalCodec<t.Codec<number | null, number | null, string, t.CodecProps>>;
|
|
74
79
|
}>>;
|
|
75
80
|
/**
|
|
76
81
|
* Config input specified when starting services
|
package/dist/types/types.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb/types';
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
3
|
import * as service_types from '@powersync/service-types';
|
|
3
4
|
import * as t from 'ts-codec';
|
|
4
5
|
export var PostImagesOption;
|
|
@@ -35,9 +36,16 @@ export var PostImagesOption;
|
|
|
35
36
|
*/
|
|
36
37
|
PostImagesOption["READ_ONLY"] = "read_only";
|
|
37
38
|
})(PostImagesOption || (PostImagesOption = {}));
|
|
39
|
+
const DEFAULT_PING_INTERVAL_SECONDS = 60;
|
|
40
|
+
const MIN_PING_INTERVAL_SECONDS = 5;
|
|
41
|
+
const MAX_PING_INTERVAL_SECONDS = 60;
|
|
38
42
|
export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(t.object({
|
|
39
43
|
// Replication specific settings
|
|
40
|
-
post_images:
|
|
44
|
+
post_images: service_types.enumLiteral('off', 'auto_configure', 'read_only').optional(),
|
|
45
|
+
/**
|
|
46
|
+
* Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
|
|
47
|
+
*/
|
|
48
|
+
heartbeat_interval_seconds: service_types.orNull(t.number).optional()
|
|
41
49
|
}));
|
|
42
50
|
/**
|
|
43
51
|
* Validate and normalize connection options.
|
|
@@ -50,7 +58,17 @@ export function normalizeConnectionConfig(options) {
|
|
|
50
58
|
...base,
|
|
51
59
|
id: options.id ?? 'default',
|
|
52
60
|
tag: options.tag ?? 'default',
|
|
53
|
-
postImages: options.post_images ?? PostImagesOption.OFF
|
|
61
|
+
postImages: options.post_images ?? PostImagesOption.OFF,
|
|
62
|
+
heartbeat_interval_seconds: normalizeHeartbeatInterval(options.heartbeat_interval_seconds)
|
|
54
63
|
};
|
|
55
64
|
}
|
|
65
|
+
function normalizeHeartbeatInterval(value) {
|
|
66
|
+
if (value == null) {
|
|
67
|
+
return DEFAULT_PING_INTERVAL_SECONDS;
|
|
68
|
+
}
|
|
69
|
+
if (!Number.isFinite(value) || value < MIN_PING_INTERVAL_SECONDS || value > MAX_PING_INTERVAL_SECONDS) {
|
|
70
|
+
throw new ServiceError(ErrorCode.PSYNC_S1109, `MongoDB connection: heartbeat_interval_seconds must be between ${MIN_PING_INTERVAL_SECONDS} and ${MAX_PING_INTERVAL_SECONDS} seconds; null or omitted defaults to ${DEFAULT_PING_INTERVAL_SECONDS}`);
|
|
71
|
+
}
|
|
72
|
+
return value;
|
|
73
|
+
}
|
|
56
74
|
//# sourceMappingURL=types.js.map
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAClE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,sCAAsC,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAED,MAAM,6BAA6B,GAAG,EAAE,CAAC;AACzC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AAqBrC,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC/G,CAAC,CAAC,MAAM,CAAC;IACP,gCAAgC;IAChC,WAAW,EAAE,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC,QAAQ,EAAE;IACvF;;OAEG;IACH,0BAA0B,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;CACtE,CAAC,CACH,CAAC;AAaF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAqC;IAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;QACzF,0BAA0B,EAAE,0BAA0B,CAAC,OAAO,CAAC,0BAA0B,CAAC;KAC3F,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,KAAgC;IAClE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,6BAA6B,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,yBAAyB,IAAI,KAAK,GAAG,yBAAyB,EAAE,CAAC;QACtG,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,kEAAkE,yBAAyB,QAAQ,yBAAyB,yCAAyC,6BAA6B,EAAE,CACrM,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-mongodb",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.0.0-dev-
|
|
4
|
+
"version": "0.0.0-dev-20260909133214",
|
|
5
5
|
"license": "FSL-1.1-ALv2",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
@@ -15,17 +15,17 @@
|
|
|
15
15
|
"bson": "^6.10.4",
|
|
16
16
|
"ts-codec": "^1.3.0",
|
|
17
17
|
"uuid": "^14.0.0",
|
|
18
|
-
"@powersync/lib-service-mongodb": "0.0.0-dev-
|
|
19
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
20
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
18
|
+
"@powersync/lib-service-mongodb": "0.0.0-dev-20260909133214",
|
|
19
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20260909133214",
|
|
20
|
+
"@powersync/service-core": "0.0.0-dev-20260909133214",
|
|
21
21
|
"@powersync/service-jsonbig": "0.17.13",
|
|
22
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
23
|
-
"@powersync/service-types": "0.
|
|
22
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20260909133214",
|
|
23
|
+
"@powersync/service-types": "0.18.0"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
27
|
-
"@powersync/service-module-mongodb-storage": "0.0.0-dev-
|
|
28
|
-
"@powersync/service-module-postgres-storage": "0.0.0-dev-
|
|
26
|
+
"@powersync/service-core-tests": "0.0.0-dev-20260909133214",
|
|
27
|
+
"@powersync/service-module-mongodb-storage": "0.0.0-dev-20260909133214",
|
|
28
|
+
"@powersync/service-module-postgres-storage": "0.0.0-dev-20260909133214"
|
|
29
29
|
},
|
|
30
30
|
"scripts": {
|
|
31
31
|
"build": "tsc -b",
|
|
@@ -39,7 +39,8 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
|
|
|
39
39
|
storageEngine: context.storageEngine,
|
|
40
40
|
metricsEngine: context.metricsEngine,
|
|
41
41
|
connectionFactory: connectionFactory,
|
|
42
|
-
rateLimiter: new MongoErrorRateLimiter()
|
|
42
|
+
rateLimiter: new MongoErrorRateLimiter(),
|
|
43
|
+
heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
|
|
43
44
|
});
|
|
44
45
|
}
|
|
45
46
|
|
|
@@ -123,15 +123,27 @@ export class ChangeStream {
|
|
|
123
123
|
|
|
124
124
|
private isDocumentDb = false;
|
|
125
125
|
private _checkpointImplementation: CheckpointImplementation | null = null;
|
|
126
|
+
/**
|
|
127
|
+
* Last persisted resume timestamp as Date.now(). This represents the equivalent timestamp from the
|
|
128
|
+
* source database, rather than the time when we persisted it.
|
|
129
|
+
*/
|
|
130
|
+
private lastPersistedResumeTimestamp = 0;
|
|
131
|
+
private lastBatchCheckpoint = 0;
|
|
126
132
|
|
|
127
133
|
constructor(options: ChangeStreamOptions) {
|
|
128
134
|
this.storage = options.storage;
|
|
129
135
|
this.metrics = options.metrics;
|
|
130
136
|
this.group_id = options.storage.replicationStreamId;
|
|
131
137
|
this.connections = options.connections;
|
|
132
|
-
this.maxAwaitTimeMS = options.maxAwaitTimeMS ?? 10_000;
|
|
133
138
|
this.snapshotChunkLength = options.snapshotChunkLength ?? 6_000;
|
|
134
139
|
this.keepaliveIntervalMs = options.keepaliveIntervalMs ?? 60_000;
|
|
140
|
+
if (options.maxAwaitTimeMS) {
|
|
141
|
+
this.maxAwaitTimeMS = options.maxAwaitTimeMS;
|
|
142
|
+
} else {
|
|
143
|
+
// Default to 10s, unless keepaliveIntervalMs is shorter. In that case, use keepaliveIntervalMs,
|
|
144
|
+
// but add a bit of a margin, so that our keepalive logic is triggered on each empty change stream batch.
|
|
145
|
+
this.maxAwaitTimeMS = Math.min(10_000, this.keepaliveIntervalMs + 100);
|
|
146
|
+
}
|
|
135
147
|
this.storageHooks = options.storageHooks;
|
|
136
148
|
this.client = this.connections.client;
|
|
137
149
|
this.defaultDb = this.connections.db;
|
|
@@ -201,7 +213,7 @@ export class ChangeStream {
|
|
|
201
213
|
this.isDocumentDb = await detectDocumentDb(this.defaultDb);
|
|
202
214
|
if (this.isDocumentDb) {
|
|
203
215
|
this.logger.warn(
|
|
204
|
-
'Azure DocumentDB support is
|
|
216
|
+
'Azure DocumentDB support is in alpha. APIs and behavior may change, and long-term stability is not yet guaranteed.'
|
|
205
217
|
);
|
|
206
218
|
}
|
|
207
219
|
this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
|
|
@@ -583,9 +595,6 @@ export class ChangeStream {
|
|
|
583
595
|
return rawChangeStream(watchDb, pipeline, {
|
|
584
596
|
batchSize: options.batchSize ?? this.snapshotChunkLength,
|
|
585
597
|
maxAwaitTimeMS,
|
|
586
|
-
// maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
|
|
587
|
-
// In that case there is no client-side wait to emulate for DocumentDB.
|
|
588
|
-
clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
|
|
589
598
|
maxTimeMS: this.changeStreamTimeout,
|
|
590
599
|
|
|
591
600
|
signal: options.signal,
|
|
@@ -598,6 +607,12 @@ export class ChangeStream {
|
|
|
598
607
|
return this.sourceRowConverter.rawToSqliteRow(row);
|
|
599
608
|
}
|
|
600
609
|
|
|
610
|
+
private async createBatchCheckpoint() {
|
|
611
|
+
const checkpoint = await this.checkpointImplementation.createBatchCheckpoint();
|
|
612
|
+
this.lastBatchCheckpoint = Date.now();
|
|
613
|
+
return checkpoint;
|
|
614
|
+
}
|
|
615
|
+
|
|
601
616
|
async streamChangesInternal() {
|
|
602
617
|
await this.ensureDetected();
|
|
603
618
|
const transactionsReplicatedMetric = this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED);
|
|
@@ -615,6 +630,7 @@ export class ChangeStream {
|
|
|
615
630
|
// We get a complete postimage for every change, so we don't need to store the current data.
|
|
616
631
|
storeCurrentData: false,
|
|
617
632
|
hooks: this.storageHooks,
|
|
633
|
+
signal: this.abortSignal,
|
|
618
634
|
tracer
|
|
619
635
|
},
|
|
620
636
|
async (batch) => {
|
|
@@ -643,7 +659,7 @@ export class ChangeStream {
|
|
|
643
659
|
// Always start with a checkpoint.
|
|
644
660
|
// This helps us to clear errors when restarting, even if there is
|
|
645
661
|
// no data to replicate.
|
|
646
|
-
let waitForCheckpointLsn: string | null = await this.
|
|
662
|
+
let waitForCheckpointLsn: string | null = await this.createBatchCheckpoint();
|
|
647
663
|
|
|
648
664
|
let splitDocument: ProjectedChangeStreamDocument | null = null;
|
|
649
665
|
|
|
@@ -663,11 +679,15 @@ export class ChangeStream {
|
|
|
663
679
|
}
|
|
664
680
|
this.touch();
|
|
665
681
|
if (events.length == 0) {
|
|
666
|
-
// No changes in this batch, but we still want to
|
|
682
|
+
// No changes in this batch, but we still want to persist progress.
|
|
667
683
|
// We do this by persisting a keepalive checkpoint.
|
|
668
684
|
// If we don't update it on empty events, we do keep consistency, but resuming the stream
|
|
669
685
|
// with old tokens may cause connection timeouts.
|
|
670
|
-
|
|
686
|
+
const hadRecentKeepalive = performance.now() - lastEmptyResume < this.keepaliveIntervalMs;
|
|
687
|
+
if (waitForCheckpointLsn == null && !hadRecentKeepalive) {
|
|
688
|
+
// Case 1: We have no changes, and we are not waiting for a checkpoint to be created,
|
|
689
|
+
// and we have not recently persisted a keepalive. Persist one now, and call setResumeLsn() below.
|
|
690
|
+
// This is the normal case for an idle stream.
|
|
671
691
|
// The implementation persists a keepalive (timestamp) or bumps the
|
|
672
692
|
// sentinel so a later event commits (sentinel). Logging is handled
|
|
673
693
|
// inside the implementation.
|
|
@@ -675,12 +695,15 @@ export class ChangeStream {
|
|
|
675
695
|
this.touch();
|
|
676
696
|
lastEmptyResume = performance.now();
|
|
677
697
|
this.replicationLag.markStarted();
|
|
678
|
-
}
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
if (performance.now() - lastEmptyResume < this.keepaliveIntervalMs) {
|
|
698
|
+
} else if (hadRecentKeepalive) {
|
|
699
|
+
// Case 2: We have no changes, and may or may not be waiting for a checkpoint to be created.
|
|
700
|
+
// We have recently persisted a keepalive.
|
|
701
|
+
// Continue waiting.
|
|
683
702
|
continue;
|
|
703
|
+
} else {
|
|
704
|
+
// Case 3: Waiting for a checkpoint; have not had a recent keepalive.
|
|
705
|
+
// We cannot call checkpointImplementation.keepalive() here, but we do call
|
|
706
|
+
// setResumeLsn() below.
|
|
684
707
|
}
|
|
685
708
|
}
|
|
686
709
|
|
|
@@ -802,7 +825,7 @@ export class ChangeStream {
|
|
|
802
825
|
if (hasBufferedChanges && waitForCheckpointLsn == null) {
|
|
803
826
|
// Buffered changes - create a new batch checkpoint to rate limit commits
|
|
804
827
|
using _ = tracer.span('source_checkpoint');
|
|
805
|
-
waitForCheckpointLsn = await this.
|
|
828
|
+
waitForCheckpointLsn = await this.createBatchCheckpoint();
|
|
806
829
|
continue;
|
|
807
830
|
} else if (waitForCheckpointLsn != null) {
|
|
808
831
|
// Skip this checkpoint - wait for the batch checkpoint.
|
|
@@ -836,7 +859,7 @@ export class ChangeStream {
|
|
|
836
859
|
) {
|
|
837
860
|
if (waitForCheckpointLsn == null) {
|
|
838
861
|
using _ = tracer.span('source_checkpoint');
|
|
839
|
-
waitForCheckpointLsn = await this.
|
|
862
|
+
waitForCheckpointLsn = await this.createBatchCheckpoint();
|
|
840
863
|
}
|
|
841
864
|
|
|
842
865
|
const rel = getMongoRelation(changeDocument.ns, this.connections.connectionTag);
|
|
@@ -909,10 +932,18 @@ export class ChangeStream {
|
|
|
909
932
|
// Batches are generally large (64MB or 6000 events, whichever comes first),
|
|
910
933
|
// so this is a good natural point to flush and mark progress.
|
|
911
934
|
// We avoid this when splitDocument is set, since we cannot resume in the middle of a split event.
|
|
912
|
-
const lsn = this.checkpointImplementation.lsnFromResumeToken(resumeToken);
|
|
935
|
+
const { lsn, timestamp } = this.checkpointImplementation.lsnFromResumeToken(resumeToken);
|
|
913
936
|
await batch.flush({ oldestUncommittedChange: this.replicationLag.oldestUncommittedChange });
|
|
914
937
|
// TODO: We should consider making this standard behavior of flush().
|
|
915
938
|
await batch.setResumeLsn(lsn);
|
|
939
|
+
|
|
940
|
+
if (timestamp != null) {
|
|
941
|
+
// Note that this timestamp provided by MongoDB is not exact - it can be around 10s behind.
|
|
942
|
+
this.lastPersistedResumeTimestamp = timestamp.getTime();
|
|
943
|
+
} else {
|
|
944
|
+
// DocumentDB: No timestamp associated with the resumeToken. Just use the current time.
|
|
945
|
+
this.lastPersistedResumeTimestamp = Date.now();
|
|
946
|
+
}
|
|
916
947
|
}
|
|
917
948
|
|
|
918
949
|
batchSpan.end();
|
|
@@ -940,6 +971,34 @@ export class ChangeStream {
|
|
|
940
971
|
return this.replicationLag.getLagMillis();
|
|
941
972
|
}
|
|
942
973
|
|
|
974
|
+
async keepAlive() {
|
|
975
|
+
// This is called on an interval of keepaliveIntervalMs.
|
|
976
|
+
|
|
977
|
+
// This writes to _powersync_checkpoints.
|
|
978
|
+
// Main use case: When there are massive bulk writes to another database or collection, the change stream may
|
|
979
|
+
// start timing out due to reading through too much data in one batch. This breaks up those bulk writes,
|
|
980
|
+
// allowing the change stream to make progress.
|
|
981
|
+
|
|
982
|
+
// When there is low replication traffic, regular calls to setResumeLsn() with recent resume tokens make this unnecessary.
|
|
983
|
+
// We track that using lastPersistedResumeTimestamp.
|
|
984
|
+
// We add a bit of a margin here, since setResumeLsn is not called on an exact interval.
|
|
985
|
+
const staleResumeLsn = Date.now() - this.lastPersistedResumeTimestamp > this.keepaliveIntervalMs * 1.1;
|
|
986
|
+
|
|
987
|
+
// When there is replication lag, that resumeToken can get too outdated. In that case, we periodically create a
|
|
988
|
+
// new back checkpoint. This interval is controlled by the frequency of the keepAlive() call,
|
|
989
|
+
// while also skipping if there was another call to createBatchCheckpoint().
|
|
990
|
+
// We use a factor of 0.9 here, to make sure this is called on every keepAlive() interval, unless there
|
|
991
|
+
// was another call to createBatchCheckpoint().
|
|
992
|
+
const staleBatchCheckpoint = Date.now() - this.lastBatchCheckpoint > this.keepaliveIntervalMs * 0.9;
|
|
993
|
+
|
|
994
|
+
// We don't use oldestUncommittedChange here, since that may be unset in some edge cases where we do need
|
|
995
|
+
// to persist new checkpoints.
|
|
996
|
+
if (staleResumeLsn && staleBatchCheckpoint) {
|
|
997
|
+
await this.ensureDetected();
|
|
998
|
+
await this.createBatchCheckpoint();
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
|
|
943
1002
|
private lastTouchedAt = performance.now();
|
|
944
1003
|
|
|
945
1004
|
private touch() {
|
|
@@ -24,7 +24,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
async keepAlive() {
|
|
27
|
-
|
|
27
|
+
await this.lastStream?.keepAlive();
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
async replicate() {
|
|
@@ -71,6 +71,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
|
|
|
71
71
|
storage: this.options.storage,
|
|
72
72
|
metrics: this.options.metrics,
|
|
73
73
|
connections: connectionManager,
|
|
74
|
+
keepaliveIntervalMs: connectionManager.options.heartbeat_interval_seconds * 1_000,
|
|
74
75
|
logger: this.logger
|
|
75
76
|
});
|
|
76
77
|
this.lastStream = stream;
|
|
@@ -212,6 +212,7 @@ export class MongoSnapshotter {
|
|
|
212
212
|
defaultSchema: this.defaultDb.databaseName,
|
|
213
213
|
storeCurrentData: false,
|
|
214
214
|
skipExistingRows: true,
|
|
215
|
+
signal: this.abortSignal,
|
|
215
216
|
tracer: new PerformanceTracer('MongoDB initial snapshot setup')
|
|
216
217
|
});
|
|
217
218
|
if (snapshotLsn == null) {
|
|
@@ -345,10 +346,10 @@ export class MongoSnapshotter {
|
|
|
345
346
|
return;
|
|
346
347
|
}
|
|
347
348
|
|
|
348
|
-
//
|
|
349
|
+
// Compact storage _after_ initial replication, but _before_ we switch to this replication stream.
|
|
349
350
|
// Keeping snapshot_done false until this completes makes this resumable after interruption.
|
|
350
351
|
// No checkpoint exists yet - storage defaults to its highest persisted op id.
|
|
351
|
-
await this.storage.
|
|
352
|
+
await this.storage.compactInitialReplication({
|
|
352
353
|
signal: this.abortSignal
|
|
353
354
|
});
|
|
354
355
|
|
|
@@ -361,7 +362,8 @@ export class MongoSnapshotter {
|
|
|
361
362
|
zeroLSN: MongoLSN.ZERO.comparable,
|
|
362
363
|
defaultSchema: this.defaultDb.databaseName,
|
|
363
364
|
storeCurrentData: false,
|
|
364
|
-
skipExistingRows: true
|
|
365
|
+
skipExistingRows: true,
|
|
366
|
+
signal: this.abortSignal
|
|
365
367
|
});
|
|
366
368
|
|
|
367
369
|
// The checkpoint here is a marker - we need to replicate up to at least this
|
|
@@ -391,6 +393,7 @@ export class MongoSnapshotter {
|
|
|
391
393
|
storeCurrentData: false,
|
|
392
394
|
skipExistingRows: true,
|
|
393
395
|
hooks: this.storageHooks,
|
|
396
|
+
signal: this.abortSignal,
|
|
394
397
|
tracer: new PerformanceTracer('MongoDB snapshot table')
|
|
395
398
|
});
|
|
396
399
|
// Get fresh table info, in case it was updated while queuing.
|
|
@@ -782,9 +785,6 @@ export class MongoSnapshotter {
|
|
|
782
785
|
return rawChangeStream(watchDb, pipeline, {
|
|
783
786
|
batchSize: options.batchSize ?? this.snapshotChunkLength,
|
|
784
787
|
maxAwaitTimeMS,
|
|
785
|
-
// maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
|
|
786
|
-
// In that case there is no client-side wait to emulate for DocumentDB.
|
|
787
|
-
clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
|
|
788
788
|
maxTimeMS: this.changeStreamTimeout,
|
|
789
789
|
signal: options.signal,
|
|
790
790
|
logger: this.logger,
|