@powersync/service-module-mongodb 0.0.0-dev-20250122110924 → 0.0.0-dev-20250214100224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -11
- package/dist/api/MongoRouteAPIAdapter.d.ts +2 -1
- package/dist/api/MongoRouteAPIAdapter.js +39 -0
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/common/MongoLSN.d.ts +31 -0
- package/dist/common/MongoLSN.js +47 -0
- package/dist/common/MongoLSN.js.map +1 -0
- package/dist/module/MongoModule.js +2 -2
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +3 -3
- package/dist/replication/ChangeStream.js +66 -22
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +5 -4
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/ChangeStreamReplicator.js +1 -0
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/dist/replication/ConnectionManagerFactory.js +2 -0
- package/dist/replication/ConnectionManagerFactory.js.map +1 -1
- package/dist/replication/MongoErrorRateLimiter.js +5 -7
- package/dist/replication/MongoErrorRateLimiter.js.map +1 -1
- package/dist/replication/MongoManager.js +10 -4
- package/dist/replication/MongoManager.js.map +1 -1
- package/dist/replication/MongoRelation.d.ts +0 -2
- package/dist/replication/MongoRelation.js +3 -15
- package/dist/replication/MongoRelation.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MongoRouteAPIAdapter.ts +41 -1
- package/src/common/MongoLSN.ts +74 -0
- package/src/replication/ChangeStream.ts +65 -28
- package/src/replication/ChangeStreamReplicationJob.ts +4 -4
- package/src/replication/MongoRelation.ts +3 -17
- package/test/src/change_stream_utils.ts +2 -2
- package/test/src/resume.test.ts +152 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isMongoServerError } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { container } from '@powersync/lib-services-framework';
|
|
3
3
|
import { replication } from '@powersync/service-core';
|
|
4
4
|
import { ChangeStream, ChangeStreamInvalidatedError } from './ChangeStream.js';
|
|
5
5
|
export class ChangeStreamReplicationJob extends replication.AbstractReplicationJob {
|
|
6
|
+
connectionFactory;
|
|
6
7
|
constructor(options) {
|
|
7
8
|
super(options);
|
|
8
9
|
this.connectionFactory = options.connectionFactory;
|
|
@@ -64,7 +65,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
|
|
|
64
65
|
if (this.abortController.signal.aborted) {
|
|
65
66
|
return;
|
|
66
67
|
}
|
|
67
|
-
this.logger.error(
|
|
68
|
+
this.logger.error(`${this.slotName} Replication error`, e);
|
|
68
69
|
if (e.cause != null) {
|
|
69
70
|
// Without this additional log, the cause may not be visible in the logs.
|
|
70
71
|
this.logger.error(`cause`, e.cause);
|
|
@@ -72,8 +73,8 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
|
|
|
72
73
|
if (e instanceof ChangeStreamInvalidatedError) {
|
|
73
74
|
throw e;
|
|
74
75
|
}
|
|
75
|
-
else if (e
|
|
76
|
-
throw new ChangeStreamInvalidatedError(e.message);
|
|
76
|
+
else if (isMongoServerError(e) && e.hasErrorLabel('NonResumableChangeStreamError')) {
|
|
77
|
+
throw new ChangeStreamInvalidatedError(e.message, e);
|
|
77
78
|
}
|
|
78
79
|
else {
|
|
79
80
|
// Report the error if relevant, before retrying
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChangeStreamReplicationJob.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"ChangeStreamReplicationJob.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,mCAAmC,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,YAAY,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAO/E,MAAM,OAAO,0BAA2B,SAAQ,WAAW,CAAC,sBAAsB;IACxE,iBAAiB,CAA2B;IAEpD,YAAY,OAA0C;QACpD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,OAAO;QACX,mBAAmB;IACrB,CAAC;IAED,KAAK,CAAC,SAAS;QACb,mBAAmB;IACrB,CAAC;IAED,IAAY,QAAQ;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAClB,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;gBACrC,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YAE3C,IAAI,CAAC,YAAY,4BAA4B,EAAE,CAAC;gBAC9C,8DAA8D;gBAC9D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC1D,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;gBAC9B,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACxC,OAAO;YACT,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,oBAAoB,EAAE,CAAC,CAAC,CAAC;YAC3D,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpB,yEAAyE;gBACzE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACtC,CAAC;YACD,IAAI,CAAC,YAAY,4BAA4B,EAAE,CAAC;gBAC9C,MAAM,CAAC,CAAC;YACV,CAAC;iBAAM,IAAI,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBACrF,MAAM,IAAI,4BAA4B,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,gDAAgD;gBAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;gBACH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -3,6 +3,7 @@ import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js';
|
|
|
3
3
|
import { MongoErrorRateLimiter } from './MongoErrorRateLimiter.js';
|
|
4
4
|
import { MongoModule } from '../module/MongoModule.js';
|
|
5
5
|
export class ChangeStreamReplicator extends replication.AbstractReplicator {
|
|
6
|
+
connectionFactory;
|
|
6
7
|
constructor(options) {
|
|
7
8
|
super(options);
|
|
8
9
|
this.connectionFactory = options.connectionFactory;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChangeStreamReplicator.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAE7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAMvD,MAAM,OAAO,sBAAuB,SAAQ,WAAW,CAAC,kBAA8C;
|
|
1
|
+
{"version":3,"file":"ChangeStreamReplicator.js","sourceRoot":"","sources":["../../src/replication/ChangeStreamReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAE7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAMvD,MAAM,OAAO,sBAAuB,SAAQ,WAAW,CAAC,kBAA8C;IACnF,iBAAiB,CAA2B;IAE7D,YAAY,OAAsC;QAChD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrD,CAAC;IAED,SAAS,CAAC,OAAqC;QAC7C,OAAO,IAAI,0BAA0B,CAAC;YACpC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC9C,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,IAAI,qBAAqB,EAAE;SACzC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,gBAAgD;QAC5D,4BAA4B;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;IACrF,CAAC;CACF"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { MongoManager } from './MongoManager.js';
|
|
3
3
|
export class ConnectionManagerFactory {
|
|
4
|
+
connectionManagers;
|
|
5
|
+
dbConnectionConfig;
|
|
4
6
|
constructor(dbConnectionConfig) {
|
|
5
7
|
this.dbConnectionConfig = dbConnectionConfig;
|
|
6
8
|
this.connectionManagers = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConnectionManagerFactory.js","sourceRoot":"","sources":["../../src/replication/ConnectionManagerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,OAAO,wBAAwB;
|
|
1
|
+
{"version":3,"file":"ConnectionManagerFactory.js","sourceRoot":"","sources":["../../src/replication/ConnectionManagerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,OAAO,wBAAwB;IAClB,kBAAkB,CAAiB;IACpC,kBAAkB,CAAkC;IAEpE,YAAY,kBAAmD;QAC7D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;IAC/B,CAAC;IAED,MAAM;QACJ,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1D,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC5D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC9C,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;QACtB,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IACjE,CAAC;CACF"}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { setTimeout } from 'timers/promises';
|
|
2
2
|
export class MongoErrorRateLimiter {
|
|
3
|
-
|
|
4
|
-
this.nextAllowed = Date.now();
|
|
5
|
-
}
|
|
3
|
+
nextAllowed = Date.now();
|
|
6
4
|
async waitUntilAllowed(options) {
|
|
7
5
|
const delay = Math.max(0, this.nextAllowed - Date.now());
|
|
8
6
|
// Minimum delay between connections, even without errors
|
|
@@ -17,18 +15,18 @@ export class MongoErrorRateLimiter {
|
|
|
17
15
|
const message = e.message ?? '';
|
|
18
16
|
if (message.includes('password authentication failed')) {
|
|
19
17
|
// Wait 15 minutes, to avoid triggering Supabase's fail2ban
|
|
20
|
-
this.setDelay(
|
|
18
|
+
this.setDelay(900_000);
|
|
21
19
|
}
|
|
22
20
|
else if (message.includes('ENOTFOUND')) {
|
|
23
21
|
// DNS lookup issue - incorrect URI or deleted instance
|
|
24
|
-
this.setDelay(
|
|
22
|
+
this.setDelay(120_000);
|
|
25
23
|
}
|
|
26
24
|
else if (message.includes('ECONNREFUSED')) {
|
|
27
25
|
// Could be fail2ban or similar
|
|
28
|
-
this.setDelay(
|
|
26
|
+
this.setDelay(120_000);
|
|
29
27
|
}
|
|
30
28
|
else {
|
|
31
|
-
this.setDelay(
|
|
29
|
+
this.setDelay(30_000);
|
|
32
30
|
}
|
|
33
31
|
}
|
|
34
32
|
setDelay(delay) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/MongoErrorRateLimiter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,MAAM,OAAO,qBAAqB;
|
|
1
|
+
{"version":3,"file":"MongoErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/MongoErrorRateLimiter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,MAAM,OAAO,qBAAqB;IAChC,WAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjC,KAAK,CAAC,gBAAgB,CAAC,OAA0D;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,WAAW,CAAC,CAAM;QAChB,6CAA6C;QAC7C,MAAM,OAAO,GAAI,CAAC,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE,CAAC;YACvD,2DAA2D;YAC3D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACzC,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YAC5C,+BAA+B;YAC/B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;CACF"}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
export class MongoManager {
|
|
3
|
+
options;
|
|
4
|
+
/**
|
|
5
|
+
* Do not use this for any transactions.
|
|
6
|
+
*/
|
|
7
|
+
client;
|
|
8
|
+
db;
|
|
3
9
|
constructor(options, overrides) {
|
|
4
10
|
this.options = options;
|
|
5
11
|
// The pool is lazy - no connections are opened until a query is performed.
|
|
@@ -10,17 +16,17 @@ export class MongoManager {
|
|
|
10
16
|
},
|
|
11
17
|
lookup: options.lookup,
|
|
12
18
|
// Time for connection to timeout
|
|
13
|
-
connectTimeoutMS:
|
|
19
|
+
connectTimeoutMS: 5_000,
|
|
14
20
|
// Time for individual requests to timeout
|
|
15
|
-
socketTimeoutMS:
|
|
21
|
+
socketTimeoutMS: 60_000,
|
|
16
22
|
// How long to wait for new primary selection
|
|
17
|
-
serverSelectionTimeoutMS:
|
|
23
|
+
serverSelectionTimeoutMS: 30_000,
|
|
18
24
|
// Avoid too many connections:
|
|
19
25
|
// 1. It can overwhelm the source database.
|
|
20
26
|
// 2. Processing too many queries in parallel can cause the process to run out of memory.
|
|
21
27
|
maxPoolSize: 8,
|
|
22
28
|
maxConnecting: 3,
|
|
23
|
-
maxIdleTimeMS:
|
|
29
|
+
maxIdleTimeMS: 60_000,
|
|
24
30
|
...overrides
|
|
25
31
|
});
|
|
26
32
|
this.db = this.client.db(options.database, {});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoManager.js","sourceRoot":"","sources":["../../src/replication/MongoManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAIvD,MAAM,OAAO,YAAY;
|
|
1
|
+
{"version":3,"file":"MongoManager.js","sourceRoot":"","sources":["../../src/replication/MongoManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAIvD,MAAM,OAAO,YAAY;IAQd;IAPT;;OAEG;IACa,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAE7B,YACS,OAAwC,EAC/C,SAAoC;QAD7B,YAAO,GAAP,OAAO,CAAiC;QAG/C,2EAA2E;QAC3E,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE;YAC/C,IAAI,EAAE;gBACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B;YAED,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,iCAAiC;YACjC,gBAAgB,EAAE,KAAK;YACvB,0CAA0C;YAC1C,eAAe,EAAE,MAAM;YACvB,6CAA6C;YAC7C,wBAAwB,EAAE,MAAM;YAEhC,8BAA8B;YAC9B,2CAA2C;YAC3C,yFAAyF;YACzF,WAAW,EAAE,CAAC;YAEd,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,MAAM;YACrB,GAAG,SAAS;SACb,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,mBAAmB;IACrB,CAAC;CACF"}
|
|
@@ -2,8 +2,6 @@ import { mongo } from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { storage } from '@powersync/service-core';
|
|
3
3
|
import { SqliteRow, SqliteValue } from '@powersync/service-sync-rules';
|
|
4
4
|
export declare function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor;
|
|
5
|
-
export declare function getMongoLsn(timestamp: mongo.Timestamp): string;
|
|
6
|
-
export declare function mongoLsnToTimestamp(lsn: string | null): mongo.BSON.Timestamp | null;
|
|
7
5
|
export declare function constructAfterRecord(document: mongo.Document): SqliteRow;
|
|
8
6
|
export declare function toMongoSyncRulesValue(data: any): SqliteValue;
|
|
9
7
|
export declare function createCheckpoint(client: mongo.MongoClient, db: mongo.Db): Promise<string>;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
3
|
-
import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
4
3
|
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
4
|
+
import { MongoLSN } from '../common/MongoLSN.js';
|
|
5
|
+
import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
5
6
|
export function getMongoRelation(source) {
|
|
6
7
|
return {
|
|
7
8
|
name: source.coll,
|
|
@@ -10,19 +11,6 @@ export function getMongoRelation(source) {
|
|
|
10
11
|
replicationColumns: [{ name: '_id' }]
|
|
11
12
|
};
|
|
12
13
|
}
|
|
13
|
-
export function getMongoLsn(timestamp) {
|
|
14
|
-
const a = timestamp.high.toString(16).padStart(8, '0');
|
|
15
|
-
const b = timestamp.low.toString(16).padStart(8, '0');
|
|
16
|
-
return a + b;
|
|
17
|
-
}
|
|
18
|
-
export function mongoLsnToTimestamp(lsn) {
|
|
19
|
-
if (lsn == null) {
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
const a = parseInt(lsn.substring(0, 8), 16);
|
|
23
|
-
const b = parseInt(lsn.substring(8, 16), 16);
|
|
24
|
-
return mongo.Timestamp.fromBits(b, a);
|
|
25
|
-
}
|
|
26
14
|
export function constructAfterRecord(document) {
|
|
27
15
|
let record = {};
|
|
28
16
|
for (let key of Object.keys(document)) {
|
|
@@ -197,7 +185,7 @@ export async function createCheckpoint(client, db) {
|
|
|
197
185
|
});
|
|
198
186
|
const time = session.operationTime;
|
|
199
187
|
// TODO: Use the above when we support custom write checkpoints
|
|
200
|
-
return
|
|
188
|
+
return new MongoLSN({ timestamp: time }).comparable;
|
|
201
189
|
}
|
|
202
190
|
finally {
|
|
203
191
|
await session.endSession();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoRelation.js","sourceRoot":"","sources":["../../src/replication/MongoRelation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAGpE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"MongoRelation.js","sourceRoot":"","sources":["../../src/replication/MongoRelation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAGpE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,MAAM,UAAU,gBAAgB,CAAC,MAAmC;IAClE,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,EAAE;QACjB,QAAQ,EAAE,MAAM,CAAC,IAAI;QACrB,kBAAkB,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;KACG,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAwB;IAC3D,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAS;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC;IACxB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,WAAW,EAAE,CAAC;QACtC,2EAA2E;QAC3E,uBAAuB;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,SAAS,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACxB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACvE,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,sFAAsF;QACtF,OAAO,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;SAAM,IAAI,IAAI,YAAY,UAAU,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,aAAa,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,WAAW,GAAG,EAAE,CAAC;AAEvB,SAAS,cAAc,CAAC,IAAS,EAAE,KAAK,GAAG,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC;IACxB,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QACxB,kDAAkD;QAClD,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,iDAAiD,WAAW,EAAE,CAAC,CAAC;IAChH,CAAC;IACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,WAAW,EAAE,CAAC;QACtC,sCAAsC;QACtC,wCAAwC;QACxC,mCAAmC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,SAAS,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACxB,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IACvD,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,OAAO,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,IAAI,YAAY,aAAa,EAAE,CAAC;QACzC,oEAAoE;QACpE,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAyB,EAAE,EAAY;IAC5E,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;IACtC,IAAI,CAAC;QACH,kFAAkF;QAClF,8EAA8E;QAC9E,iEAAiE;QACjE,MAAM,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,gBAAgB,CAC1D;YACE,GAAG,EAAE,YAAmB;SACzB,EACD;YACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;SACf,EACD;YACE,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,OAAO;YACvB,OAAO;SACR,CACF,CAAC;QACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;QACpC,+DAA+D;QAC/D,OAAO,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC;IACtD,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.0.0-dev-
|
|
5
|
+
"version": "0.0.0-dev-20250214100224",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-Apache-2.0",
|
|
8
8
|
"type": "module",
|
|
@@ -25,18 +25,18 @@
|
|
|
25
25
|
"bson": "^6.8.0",
|
|
26
26
|
"ts-codec": "^1.3.0",
|
|
27
27
|
"uuid": "^9.0.1",
|
|
28
|
-
"@powersync/lib-services-framework": "0.
|
|
29
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
28
|
+
"@powersync/lib-services-framework": "0.5.1",
|
|
29
|
+
"@powersync/service-core": "0.0.0-dev-20250214100224",
|
|
30
30
|
"@powersync/service-jsonbig": "0.17.10",
|
|
31
|
-
"@powersync/service-sync-rules": "0.
|
|
32
|
-
"@powersync/service-types": "0.
|
|
33
|
-
"@powersync/lib-service-mongodb": "0.
|
|
31
|
+
"@powersync/service-sync-rules": "0.23.4",
|
|
32
|
+
"@powersync/service-types": "0.8.0",
|
|
33
|
+
"@powersync/lib-service-mongodb": "0.4.1"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
36
|
"@types/uuid": "^9.0.4",
|
|
37
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
38
|
-
"@powersync/service-module-mongodb-storage": "0.0.0-dev-
|
|
39
|
-
"@powersync/service-module-postgres-storage": "0.0.0-dev-
|
|
37
|
+
"@powersync/service-core-tests": "0.0.0-dev-20250214100224",
|
|
38
|
+
"@powersync/service-module-mongodb-storage": "0.0.0-dev-20250214100224",
|
|
39
|
+
"@powersync/service-module-postgres-storage": "0.0.0-dev-20250214100224"
|
|
40
40
|
},
|
|
41
41
|
"scripts": {
|
|
42
42
|
"build": "tsc -b",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
|
-
import { api, ParseSyncRulesOptions, SourceTable } from '@powersync/service-core';
|
|
3
|
+
import { api, ParseSyncRulesOptions, ReplicationHeadCallback, SourceTable } from '@powersync/service-core';
|
|
4
4
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
5
5
|
import * as service_types from '@powersync/service-types';
|
|
6
6
|
|
|
@@ -9,6 +9,8 @@ import { constructAfterRecord, createCheckpoint } from '../replication/MongoRela
|
|
|
9
9
|
import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js';
|
|
10
10
|
import * as types from '../types/types.js';
|
|
11
11
|
import { escapeRegExp } from '../utils.js';
|
|
12
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
13
|
+
import { MongoLSN } from '../common/MongoLSN.js';
|
|
12
14
|
|
|
13
15
|
export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
14
16
|
protected client: mongo.MongoClient;
|
|
@@ -208,6 +210,44 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
208
210
|
return createCheckpoint(this.client, this.db);
|
|
209
211
|
}
|
|
210
212
|
|
|
213
|
+
async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
|
|
214
|
+
const session = this.client.startSession();
|
|
215
|
+
try {
|
|
216
|
+
await this.db.command({ hello: 1 }, { session });
|
|
217
|
+
const head = session.clusterTime?.clusterTime;
|
|
218
|
+
if (head == null) {
|
|
219
|
+
throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const r = await callback(new MongoLSN({ timestamp: head }).comparable);
|
|
223
|
+
|
|
224
|
+
// Trigger a change on the changestream.
|
|
225
|
+
await this.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
|
|
226
|
+
{
|
|
227
|
+
_id: 'checkpoint' as any
|
|
228
|
+
},
|
|
229
|
+
{
|
|
230
|
+
$inc: { i: 1 }
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
upsert: true,
|
|
234
|
+
returnDocument: 'after',
|
|
235
|
+
session
|
|
236
|
+
}
|
|
237
|
+
);
|
|
238
|
+
const time = session.operationTime!;
|
|
239
|
+
if (time == null) {
|
|
240
|
+
throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
|
|
241
|
+
} else if (time.lt(head)) {
|
|
242
|
+
throw new ServiceAssertionError(`operationTime must be > clusterTime`);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
return r;
|
|
246
|
+
} finally {
|
|
247
|
+
await session.endSession();
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
211
251
|
async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
|
|
212
252
|
const sampleSize = 50;
|
|
213
253
|
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { storage } from '@powersync/service-core';
|
|
3
|
+
|
|
4
|
+
export type MongoLSNSpecification = {
|
|
5
|
+
timestamp: mongo.Timestamp;
|
|
6
|
+
/**
|
|
7
|
+
* The ResumeToken type here is an alias for `unknown`.
|
|
8
|
+
* The docs mention the contents should be of the form:
|
|
9
|
+
* ```typescript
|
|
10
|
+
* {
|
|
11
|
+
* "_data" : <BinData|string>
|
|
12
|
+
* }
|
|
13
|
+
* ```
|
|
14
|
+
* We use BSON serialization to store the resume token.
|
|
15
|
+
*/
|
|
16
|
+
resume_token?: mongo.ResumeToken;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export const ZERO_LSN = '0000000000000000';
|
|
20
|
+
|
|
21
|
+
const DELIMINATOR = '|';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Represent a Logical Sequence Number (LSN) for MongoDB replication sources.
|
|
25
|
+
* This stores a combination of the cluster timestamp and optional Change Stream resume token.
|
|
26
|
+
*/
|
|
27
|
+
export class MongoLSN {
|
|
28
|
+
static fromSerialized(comparable: string): MongoLSN {
|
|
29
|
+
return new MongoLSN(MongoLSN.deserialize(comparable));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private static deserialize(comparable: string): MongoLSNSpecification {
|
|
33
|
+
const [timestampString, resumeString] = comparable.split(DELIMINATOR);
|
|
34
|
+
|
|
35
|
+
const a = parseInt(timestampString.substring(0, 8), 16);
|
|
36
|
+
const b = parseInt(timestampString.substring(8, 16), 16);
|
|
37
|
+
|
|
38
|
+
return {
|
|
39
|
+
timestamp: mongo.Timestamp.fromBits(b, a),
|
|
40
|
+
resume_token: resumeString ? storage.deserializeBson(Buffer.from(resumeString, 'base64')).resumeToken : null
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
static ZERO = MongoLSN.fromSerialized(ZERO_LSN);
|
|
45
|
+
|
|
46
|
+
constructor(protected options: MongoLSNSpecification) {}
|
|
47
|
+
|
|
48
|
+
get timestamp() {
|
|
49
|
+
return this.options.timestamp;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
get resumeToken() {
|
|
53
|
+
return this.options.resume_token;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
get comparable() {
|
|
57
|
+
const { timestamp, resumeToken } = this;
|
|
58
|
+
|
|
59
|
+
const a = timestamp.high.toString(16).padStart(8, '0');
|
|
60
|
+
const b = timestamp.low.toString(16).padStart(8, '0');
|
|
61
|
+
|
|
62
|
+
const segments = [`${a}${b}`];
|
|
63
|
+
|
|
64
|
+
if (resumeToken) {
|
|
65
|
+
segments.push(storage.serializeBson({ resumeToken }).toString('base64'));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return segments.join(DELIMINATOR);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
toString() {
|
|
72
|
+
return this.comparable;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import {
|
|
3
3
|
container,
|
|
4
|
+
DatabaseConnectionError,
|
|
4
5
|
ErrorCode,
|
|
5
6
|
logger,
|
|
6
7
|
ReplicationAbortedError,
|
|
@@ -9,20 +10,13 @@ import {
|
|
|
9
10
|
} from '@powersync/lib-services-framework';
|
|
10
11
|
import { Metrics, SaveOperationTag, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core';
|
|
11
12
|
import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
|
|
13
|
+
import { MongoLSN } from '../common/MongoLSN.js';
|
|
12
14
|
import { PostImagesOption } from '../types/types.js';
|
|
13
15
|
import { escapeRegExp } from '../utils.js';
|
|
14
16
|
import { MongoManager } from './MongoManager.js';
|
|
15
|
-
import {
|
|
16
|
-
constructAfterRecord,
|
|
17
|
-
createCheckpoint,
|
|
18
|
-
getMongoLsn,
|
|
19
|
-
getMongoRelation,
|
|
20
|
-
mongoLsnToTimestamp
|
|
21
|
-
} from './MongoRelation.js';
|
|
17
|
+
import { constructAfterRecord, createCheckpoint, getMongoRelation } from './MongoRelation.js';
|
|
22
18
|
import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
23
19
|
|
|
24
|
-
export const ZERO_LSN = '0000000000000000';
|
|
25
|
-
|
|
26
20
|
export interface ChangeStreamOptions {
|
|
27
21
|
connections: MongoManager;
|
|
28
22
|
storage: storage.SyncRulesBucketStorage;
|
|
@@ -41,9 +35,9 @@ interface InitResult {
|
|
|
41
35
|
* * Some change stream documents do not have postImages.
|
|
42
36
|
* * startAfter/resumeToken is not valid anymore.
|
|
43
37
|
*/
|
|
44
|
-
export class ChangeStreamInvalidatedError extends
|
|
45
|
-
constructor(message: string) {
|
|
46
|
-
super(message);
|
|
38
|
+
export class ChangeStreamInvalidatedError extends DatabaseConnectionError {
|
|
39
|
+
constructor(message: string, cause: any) {
|
|
40
|
+
super(ErrorCode.PSYNC_S1344, message, cause);
|
|
47
41
|
}
|
|
48
42
|
}
|
|
49
43
|
|
|
@@ -164,7 +158,7 @@ export class ChangeStream {
|
|
|
164
158
|
|
|
165
159
|
async estimatedCount(table: storage.SourceTable): Promise<string> {
|
|
166
160
|
const db = this.client.db(table.schema);
|
|
167
|
-
const count = db.collection(table.table).estimatedDocumentCount();
|
|
161
|
+
const count = await db.collection(table.table).estimatedDocumentCount();
|
|
168
162
|
return `~${count}`;
|
|
169
163
|
}
|
|
170
164
|
|
|
@@ -207,7 +201,7 @@ export class ChangeStream {
|
|
|
207
201
|
const session = await this.client.startSession();
|
|
208
202
|
try {
|
|
209
203
|
await this.storage.startBatch(
|
|
210
|
-
{ zeroLSN:
|
|
204
|
+
{ zeroLSN: MongoLSN.ZERO.comparable, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
|
|
211
205
|
async (batch) => {
|
|
212
206
|
// Start by resolving all tables.
|
|
213
207
|
// This checks postImage configuration, and that should fail as
|
|
@@ -220,12 +214,12 @@ export class ChangeStream {
|
|
|
220
214
|
|
|
221
215
|
for (let table of allSourceTables) {
|
|
222
216
|
await this.snapshotTable(batch, table, session);
|
|
223
|
-
await batch.markSnapshotDone([table],
|
|
217
|
+
await batch.markSnapshotDone([table], MongoLSN.ZERO.comparable);
|
|
224
218
|
|
|
225
219
|
await touch();
|
|
226
220
|
}
|
|
227
221
|
|
|
228
|
-
const lsn =
|
|
222
|
+
const { comparable: lsn } = new MongoLSN({ timestamp: snapshotTime });
|
|
229
223
|
logger.info(`Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`);
|
|
230
224
|
await batch.commit(lsn);
|
|
231
225
|
}
|
|
@@ -298,6 +292,7 @@ export class ChangeStream {
|
|
|
298
292
|
logger.info(`Replicating ${table.qualifiedName}`);
|
|
299
293
|
const estimatedCount = await this.estimatedCount(table);
|
|
300
294
|
let at = 0;
|
|
295
|
+
let lastLogIndex = 0;
|
|
301
296
|
|
|
302
297
|
const db = this.client.db(table.schema);
|
|
303
298
|
const collection = db.collection(table.table);
|
|
@@ -310,8 +305,6 @@ export class ChangeStream {
|
|
|
310
305
|
throw new ReplicationAbortedError(`Aborted initial replication`);
|
|
311
306
|
}
|
|
312
307
|
|
|
313
|
-
at += 1;
|
|
314
|
-
|
|
315
308
|
const record = constructAfterRecord(document);
|
|
316
309
|
|
|
317
310
|
// This auto-flushes when the batch reaches its size limit
|
|
@@ -325,6 +318,10 @@ export class ChangeStream {
|
|
|
325
318
|
});
|
|
326
319
|
|
|
327
320
|
at += 1;
|
|
321
|
+
if (at - lastLogIndex >= 5000) {
|
|
322
|
+
logger.info(`[${this.group_id}] Replicating ${table.qualifiedName} ${at}/${estimatedCount}`);
|
|
323
|
+
lastLogIndex = at;
|
|
324
|
+
}
|
|
328
325
|
Metrics.getInstance().rows_replicated_total.add(1);
|
|
329
326
|
|
|
330
327
|
await touch();
|
|
@@ -513,7 +510,7 @@ export class ChangeStream {
|
|
|
513
510
|
e.codeName == 'NoMatchingDocument' &&
|
|
514
511
|
e.errmsg?.includes('post-image was not found')
|
|
515
512
|
) {
|
|
516
|
-
throw new ChangeStreamInvalidatedError(e.errmsg);
|
|
513
|
+
throw new ChangeStreamInvalidatedError(e.errmsg, e);
|
|
517
514
|
}
|
|
518
515
|
throw e;
|
|
519
516
|
}
|
|
@@ -524,10 +521,13 @@ export class ChangeStream {
|
|
|
524
521
|
await this.storage.autoActivate();
|
|
525
522
|
|
|
526
523
|
await this.storage.startBatch(
|
|
527
|
-
{ zeroLSN:
|
|
524
|
+
{ zeroLSN: MongoLSN.ZERO.comparable, defaultSchema: this.defaultDb.databaseName, storeCurrentData: false },
|
|
528
525
|
async (batch) => {
|
|
529
|
-
const
|
|
530
|
-
const
|
|
526
|
+
const { lastCheckpointLsn } = batch;
|
|
527
|
+
const lastLsn = lastCheckpointLsn ? MongoLSN.fromSerialized(lastCheckpointLsn) : null;
|
|
528
|
+
const startAfter = lastLsn?.timestamp;
|
|
529
|
+
const resumeAfter = lastLsn?.resumeToken;
|
|
530
|
+
|
|
531
531
|
logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`);
|
|
532
532
|
|
|
533
533
|
const filters = this.getSourceNamespaceFilters();
|
|
@@ -551,12 +551,21 @@ export class ChangeStream {
|
|
|
551
551
|
}
|
|
552
552
|
|
|
553
553
|
const streamOptions: mongo.ChangeStreamOptions = {
|
|
554
|
-
startAtOperationTime: startAfter,
|
|
555
554
|
showExpandedEvents: true,
|
|
556
555
|
useBigInt64: true,
|
|
557
556
|
maxAwaitTimeMS: 200,
|
|
558
557
|
fullDocument: fullDocument
|
|
559
558
|
};
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Only one of these options can be supplied at a time.
|
|
562
|
+
*/
|
|
563
|
+
if (resumeAfter) {
|
|
564
|
+
streamOptions.resumeAfter = resumeAfter;
|
|
565
|
+
} else {
|
|
566
|
+
streamOptions.startAtOperationTime = startAfter;
|
|
567
|
+
}
|
|
568
|
+
|
|
560
569
|
let stream: mongo.ChangeStream<mongo.Document>;
|
|
561
570
|
if (filters.multipleDatabases) {
|
|
562
571
|
// Requires readAnyDatabase@admin on Atlas
|
|
@@ -576,7 +585,7 @@ export class ChangeStream {
|
|
|
576
585
|
});
|
|
577
586
|
|
|
578
587
|
// Always start with a checkpoint.
|
|
579
|
-
// This helps us to clear
|
|
588
|
+
// This helps us to clear errors when restarting, even if there is
|
|
580
589
|
// no data to replicate.
|
|
581
590
|
let waitForCheckpointLsn: string | null = await createCheckpoint(this.client, this.defaultDb);
|
|
582
591
|
|
|
@@ -589,6 +598,11 @@ export class ChangeStream {
|
|
|
589
598
|
|
|
590
599
|
const originalChangeDocument = await stream.tryNext();
|
|
591
600
|
|
|
601
|
+
// The stream was closed, we will only ever receive `null` from it
|
|
602
|
+
if (!originalChangeDocument && stream.closed) {
|
|
603
|
+
break;
|
|
604
|
+
}
|
|
605
|
+
|
|
592
606
|
if (originalChangeDocument == null || this.abort_signal.aborted) {
|
|
593
607
|
continue;
|
|
594
608
|
}
|
|
@@ -623,15 +637,38 @@ export class ChangeStream {
|
|
|
623
637
|
throw new ReplicationAssertionError(`Incomplete splitEvent: ${JSON.stringify(splitDocument.splitEvent)}`);
|
|
624
638
|
}
|
|
625
639
|
|
|
626
|
-
// console.log('event', changeDocument);
|
|
627
|
-
|
|
628
640
|
if (
|
|
629
641
|
(changeDocument.operationType == 'insert' ||
|
|
630
642
|
changeDocument.operationType == 'update' ||
|
|
631
|
-
changeDocument.operationType == 'replace'
|
|
643
|
+
changeDocument.operationType == 'replace' ||
|
|
644
|
+
changeDocument.operationType == 'drop') &&
|
|
632
645
|
changeDocument.ns.coll == CHECKPOINTS_COLLECTION
|
|
633
646
|
) {
|
|
634
|
-
|
|
647
|
+
/**
|
|
648
|
+
* Dropping the database does not provide an `invalidate` event.
|
|
649
|
+
* We typically would receive `drop` events for the collection which we
|
|
650
|
+
* would process below.
|
|
651
|
+
*
|
|
652
|
+
* However we don't commit the LSN after collections are dropped.
|
|
653
|
+
* The prevents the `startAfter` or `resumeToken` from advancing past the drop events.
|
|
654
|
+
* The stream also closes after the drop events.
|
|
655
|
+
* This causes an infinite loop of processing the collection drop events.
|
|
656
|
+
*
|
|
657
|
+
* This check here invalidates the change stream if our `_checkpoints` collection
|
|
658
|
+
* is dropped. This allows for detecting when the DB is dropped.
|
|
659
|
+
*/
|
|
660
|
+
if (changeDocument.operationType == 'drop') {
|
|
661
|
+
throw new ChangeStreamInvalidatedError(
|
|
662
|
+
'Internal collections have been dropped',
|
|
663
|
+
new Error('_checkpoints collection was dropped')
|
|
664
|
+
);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
const { comparable: lsn } = new MongoLSN({
|
|
668
|
+
timestamp: changeDocument.clusterTime!,
|
|
669
|
+
resume_token: changeDocument._id
|
|
670
|
+
});
|
|
671
|
+
|
|
635
672
|
if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
|
|
636
673
|
waitForCheckpointLsn = null;
|
|
637
674
|
}
|