@powersync/service-module-mongodb 0.0.0-dev-20250122110924 → 0.0.0-dev-20250227082606
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -11
- package/dist/api/MongoRouteAPIAdapter.d.ts +2 -1
- package/dist/api/MongoRouteAPIAdapter.js +39 -0
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/common/MongoLSN.d.ts +31 -0
- package/dist/common/MongoLSN.js +47 -0
- package/dist/common/MongoLSN.js.map +1 -0
- package/dist/module/MongoModule.js +2 -2
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +4 -3
- package/dist/replication/ChangeStream.js +130 -52
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.js +7 -6
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/ChangeStreamReplicator.js +1 -0
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/dist/replication/ConnectionManagerFactory.js +2 -0
- package/dist/replication/ConnectionManagerFactory.js.map +1 -1
- package/dist/replication/MongoErrorRateLimiter.js +5 -7
- package/dist/replication/MongoErrorRateLimiter.js.map +1 -1
- package/dist/replication/MongoManager.d.ts +0 -3
- package/dist/replication/MongoManager.js +9 -4
- package/dist/replication/MongoManager.js.map +1 -1
- package/dist/replication/MongoRelation.d.ts +4 -2
- package/dist/replication/MongoRelation.js +10 -15
- package/dist/replication/MongoRelation.js.map +1 -1
- package/package.json +10 -10
- package/src/api/MongoRouteAPIAdapter.ts +41 -1
- package/src/common/MongoLSN.ts +74 -0
- package/src/replication/ChangeStream.ts +138 -57
- package/src/replication/ChangeStreamReplicationJob.ts +6 -6
- package/src/replication/MongoManager.ts +4 -3
- package/src/replication/MongoRelation.ts +10 -16
- package/test/src/change_stream.test.ts +6 -0
- package/test/src/change_stream_utils.ts +9 -8
- package/test/src/mongo_test.test.ts +47 -12
- package/test/src/resume.test.ts +152 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { BSON_DESERIALIZE_DATA_OPTIONS } from '@powersync/service-core';
|
|
2
3
|
export class MongoManager {
|
|
4
|
+
options;
|
|
5
|
+
client;
|
|
6
|
+
db;
|
|
3
7
|
constructor(options, overrides) {
|
|
4
8
|
this.options = options;
|
|
5
9
|
// The pool is lazy - no connections are opened until a query is performed.
|
|
@@ -10,17 +14,18 @@ export class MongoManager {
|
|
|
10
14
|
},
|
|
11
15
|
lookup: options.lookup,
|
|
12
16
|
// Time for connection to timeout
|
|
13
|
-
connectTimeoutMS:
|
|
17
|
+
connectTimeoutMS: 5_000,
|
|
14
18
|
// Time for individual requests to timeout
|
|
15
|
-
socketTimeoutMS:
|
|
19
|
+
socketTimeoutMS: 60_000,
|
|
16
20
|
// How long to wait for new primary selection
|
|
17
|
-
serverSelectionTimeoutMS:
|
|
21
|
+
serverSelectionTimeoutMS: 30_000,
|
|
18
22
|
// Avoid too many connections:
|
|
19
23
|
// 1. It can overwhelm the source database.
|
|
20
24
|
// 2. Processing too many queries in parallel can cause the process to run out of memory.
|
|
21
25
|
maxPoolSize: 8,
|
|
22
26
|
maxConnecting: 3,
|
|
23
|
-
maxIdleTimeMS:
|
|
27
|
+
maxIdleTimeMS: 60_000,
|
|
28
|
+
...BSON_DESERIALIZE_DATA_OPTIONS,
|
|
24
29
|
...overrides
|
|
25
30
|
});
|
|
26
31
|
this.db = this.client.db(options.database, {});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoManager.js","sourceRoot":"","sources":["../../src/replication/MongoManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoManager.js","sourceRoot":"","sources":["../../src/replication/MongoManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAGvD,OAAO,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAC;AAExE,MAAM,OAAO,YAAY;IAKd;IAJO,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAE7B,YACS,OAAwC,EAC/C,SAAoC;QAD7B,YAAO,GAAP,OAAO,CAAiC;QAG/C,2EAA2E;QAC3E,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE;YAC/C,IAAI,EAAE;gBACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B;YAED,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,iCAAiC;YACjC,gBAAgB,EAAE,KAAK;YACvB,0CAA0C;YAC1C,eAAe,EAAE,MAAM;YACvB,6CAA6C;YAC7C,wBAAwB,EAAE,MAAM;YAEhC,8BAA8B;YAC9B,2CAA2C;YAC3C,yFAAyF;YACzF,WAAW,EAAE,CAAC;YAEd,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,MAAM;YAErB,GAAG,6BAA6B;YAEhC,GAAG,SAAS;SACb,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,mBAAmB;IACrB,CAAC;CACF"}
|
|
@@ -2,8 +2,10 @@ import { mongo } from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { storage } from '@powersync/service-core';
|
|
3
3
|
import { SqliteRow, SqliteValue } from '@powersync/service-sync-rules';
|
|
4
4
|
export declare function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor;
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
/**
|
|
6
|
+
* For in-memory cache only.
|
|
7
|
+
*/
|
|
8
|
+
export declare function getCacheIdentifier(source: storage.SourceEntityDescriptor): string;
|
|
7
9
|
export declare function constructAfterRecord(document: mongo.Document): SqliteRow;
|
|
8
10
|
export declare function toMongoSyncRulesValue(data: any): SqliteValue;
|
|
9
11
|
export declare function createCheckpoint(client: mongo.MongoClient, db: mongo.Db): Promise<string>;
|
|
@@ -1,27 +1,22 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
3
|
-
import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
4
3
|
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
4
|
+
import { MongoLSN } from '../common/MongoLSN.js';
|
|
5
|
+
import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
|
|
5
6
|
export function getMongoRelation(source) {
|
|
6
7
|
return {
|
|
7
8
|
name: source.coll,
|
|
8
9
|
schema: source.db,
|
|
9
|
-
|
|
10
|
+
// Not relevant for MongoDB - we use db + coll name as the identifier
|
|
11
|
+
objectId: undefined,
|
|
10
12
|
replicationColumns: [{ name: '_id' }]
|
|
11
13
|
};
|
|
12
14
|
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
}
|
|
18
|
-
export function mongoLsnToTimestamp(lsn) {
|
|
19
|
-
if (lsn == null) {
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
const a = parseInt(lsn.substring(0, 8), 16);
|
|
23
|
-
const b = parseInt(lsn.substring(8, 16), 16);
|
|
24
|
-
return mongo.Timestamp.fromBits(b, a);
|
|
15
|
+
/**
|
|
16
|
+
* For in-memory cache only.
|
|
17
|
+
*/
|
|
18
|
+
export function getCacheIdentifier(source) {
|
|
19
|
+
return `${source.schema}.${source.name}`;
|
|
25
20
|
}
|
|
26
21
|
export function constructAfterRecord(document) {
|
|
27
22
|
let record = {};
|
|
@@ -197,7 +192,7 @@ export async function createCheckpoint(client, db) {
|
|
|
197
192
|
});
|
|
198
193
|
const time = session.operationTime;
|
|
199
194
|
// TODO: Use the above when we support custom write checkpoints
|
|
200
|
-
return
|
|
195
|
+
return new MongoLSN({ timestamp: time }).comparable;
|
|
201
196
|
}
|
|
202
197
|
finally {
|
|
203
198
|
await session.endSession();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoRelation.js","sourceRoot":"","sources":["../../src/replication/MongoRelation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAGpE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"MongoRelation.js","sourceRoot":"","sources":["../../src/replication/MongoRelation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAGpE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,MAAM,UAAU,gBAAgB,CAAC,MAAmC;IAClE,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,EAAE;QACjB,qEAAqE;QACrE,QAAQ,EAAE,SAAS;QACnB,kBAAkB,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;KACG,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAsC;IACvE,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAwB;IAC3D,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAS;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC;IACxB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,WAAW,EAAE,CAAC;QACtC,2EAA2E;QAC3E,uBAAuB;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,SAAS,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACxB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACvE,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,sFAAsF;QACtF,OAAO,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;SAAM,IAAI,IAAI,YAAY,UAAU,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,aAAa,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,WAAW,GAAG,EAAE,CAAC;AAEvB,SAAS,cAAc,CAAC,IAAS,EAAE,KAAK,GAAG,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC;IACxB,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QACxB,kDAAkD;QAClD,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,iDAAiD,WAAW,EAAE,CAAC,CAAC;IAChH,CAAC;IACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,WAAW,EAAE,CAAC;QACtC,sCAAsC;QACtC,wCAAwC;QACxC,mCAAmC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,SAAS,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACxB,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IACvD,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,OAAO,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,IAAI,YAAY,aAAa,EAAE,CAAC;QACzC,oEAAoE;QACpE,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE,CAAC;QACnC,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAyB,EAAE,EAAY;IAC5E,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;IACtC,IAAI,CAAC;QACH,kFAAkF;QAClF,8EAA8E;QAC9E,iEAAiE;QACjE,MAAM,EAAE,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC,gBAAgB,CAC1D;YACE,GAAG,EAAE,YAAmB;SACzB,EACD;YACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;SACf,EACD;YACE,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,OAAO;YACvB,OAAO;SACR,CACF,CAAC;QACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;QACpC,+DAA+D;QAC/D,OAAO,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC;IACtD,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.0.0-dev-
|
|
5
|
+
"version": "0.0.0-dev-20250227082606",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-Apache-2.0",
|
|
8
8
|
"type": "module",
|
|
@@ -22,21 +22,21 @@
|
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"bson": "^6.
|
|
25
|
+
"bson": "^6.10.3",
|
|
26
26
|
"ts-codec": "^1.3.0",
|
|
27
27
|
"uuid": "^9.0.1",
|
|
28
|
-
"@powersync/lib-services-framework": "0.
|
|
29
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
28
|
+
"@powersync/lib-services-framework": "0.5.3",
|
|
29
|
+
"@powersync/service-core": "0.0.0-dev-20250227082606",
|
|
30
30
|
"@powersync/service-jsonbig": "0.17.10",
|
|
31
|
-
"@powersync/service-sync-rules": "0.
|
|
32
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
33
|
-
"@powersync/lib-service-mongodb": "0.0.0-dev-
|
|
31
|
+
"@powersync/service-sync-rules": "0.24.0",
|
|
32
|
+
"@powersync/service-types": "0.0.0-dev-20250227082606",
|
|
33
|
+
"@powersync/lib-service-mongodb": "0.0.0-dev-20250227082606"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
36
|
"@types/uuid": "^9.0.4",
|
|
37
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
38
|
-
"@powersync/service-module-mongodb-storage": "0.0.0-dev-
|
|
39
|
-
"@powersync/service-module-postgres-storage": "0.0.0-dev-
|
|
37
|
+
"@powersync/service-core-tests": "0.0.0-dev-20250227082606",
|
|
38
|
+
"@powersync/service-module-mongodb-storage": "0.0.0-dev-20250227082606",
|
|
39
|
+
"@powersync/service-module-postgres-storage": "0.0.0-dev-20250227082606"
|
|
40
40
|
},
|
|
41
41
|
"scripts": {
|
|
42
42
|
"build": "tsc -b",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
|
-
import { api, ParseSyncRulesOptions, SourceTable } from '@powersync/service-core';
|
|
3
|
+
import { api, ParseSyncRulesOptions, ReplicationHeadCallback, SourceTable } from '@powersync/service-core';
|
|
4
4
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
5
5
|
import * as service_types from '@powersync/service-types';
|
|
6
6
|
|
|
@@ -9,6 +9,8 @@ import { constructAfterRecord, createCheckpoint } from '../replication/MongoRela
|
|
|
9
9
|
import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js';
|
|
10
10
|
import * as types from '../types/types.js';
|
|
11
11
|
import { escapeRegExp } from '../utils.js';
|
|
12
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
13
|
+
import { MongoLSN } from '../common/MongoLSN.js';
|
|
12
14
|
|
|
13
15
|
export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
14
16
|
protected client: mongo.MongoClient;
|
|
@@ -208,6 +210,44 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
208
210
|
return createCheckpoint(this.client, this.db);
|
|
209
211
|
}
|
|
210
212
|
|
|
213
|
+
async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
|
|
214
|
+
const session = this.client.startSession();
|
|
215
|
+
try {
|
|
216
|
+
await this.db.command({ hello: 1 }, { session });
|
|
217
|
+
const head = session.clusterTime?.clusterTime;
|
|
218
|
+
if (head == null) {
|
|
219
|
+
throw new ServiceAssertionError(`clusterTime not available for write checkpoint`);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const r = await callback(new MongoLSN({ timestamp: head }).comparable);
|
|
223
|
+
|
|
224
|
+
// Trigger a change on the changestream.
|
|
225
|
+
await this.db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
|
|
226
|
+
{
|
|
227
|
+
_id: 'checkpoint' as any
|
|
228
|
+
},
|
|
229
|
+
{
|
|
230
|
+
$inc: { i: 1 }
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
upsert: true,
|
|
234
|
+
returnDocument: 'after',
|
|
235
|
+
session
|
|
236
|
+
}
|
|
237
|
+
);
|
|
238
|
+
const time = session.operationTime!;
|
|
239
|
+
if (time == null) {
|
|
240
|
+
throw new ServiceAssertionError(`operationTime not available for write checkpoint`);
|
|
241
|
+
} else if (time.lt(head)) {
|
|
242
|
+
throw new ServiceAssertionError(`operationTime must be > clusterTime`);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
return r;
|
|
246
|
+
} finally {
|
|
247
|
+
await session.endSession();
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
211
251
|
async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
|
|
212
252
|
const sampleSize = 50;
|
|
213
253
|
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { storage } from '@powersync/service-core';
|
|
3
|
+
|
|
4
|
+
export type MongoLSNSpecification = {
|
|
5
|
+
timestamp: mongo.Timestamp;
|
|
6
|
+
/**
|
|
7
|
+
* The ResumeToken type here is an alias for `unknown`.
|
|
8
|
+
* The docs mention the contents should be of the form:
|
|
9
|
+
* ```typescript
|
|
10
|
+
* {
|
|
11
|
+
* "_data" : <BinData|string>
|
|
12
|
+
* }
|
|
13
|
+
* ```
|
|
14
|
+
* We use BSON serialization to store the resume token.
|
|
15
|
+
*/
|
|
16
|
+
resume_token?: mongo.ResumeToken;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export const ZERO_LSN = '0000000000000000';
|
|
20
|
+
|
|
21
|
+
const DELIMINATOR = '|';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Represent a Logical Sequence Number (LSN) for MongoDB replication sources.
|
|
25
|
+
* This stores a combination of the cluster timestamp and optional Change Stream resume token.
|
|
26
|
+
*/
|
|
27
|
+
export class MongoLSN {
|
|
28
|
+
static fromSerialized(comparable: string): MongoLSN {
|
|
29
|
+
return new MongoLSN(MongoLSN.deserialize(comparable));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private static deserialize(comparable: string): MongoLSNSpecification {
|
|
33
|
+
const [timestampString, resumeString] = comparable.split(DELIMINATOR);
|
|
34
|
+
|
|
35
|
+
const a = parseInt(timestampString.substring(0, 8), 16);
|
|
36
|
+
const b = parseInt(timestampString.substring(8, 16), 16);
|
|
37
|
+
|
|
38
|
+
return {
|
|
39
|
+
timestamp: mongo.Timestamp.fromBits(b, a),
|
|
40
|
+
resume_token: resumeString ? storage.deserializeBson(Buffer.from(resumeString, 'base64')).resumeToken : null
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
static ZERO = MongoLSN.fromSerialized(ZERO_LSN);
|
|
45
|
+
|
|
46
|
+
constructor(protected options: MongoLSNSpecification) {}
|
|
47
|
+
|
|
48
|
+
get timestamp() {
|
|
49
|
+
return this.options.timestamp;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
get resumeToken() {
|
|
53
|
+
return this.options.resume_token;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
get comparable() {
|
|
57
|
+
const { timestamp, resumeToken } = this;
|
|
58
|
+
|
|
59
|
+
const a = timestamp.high.toString(16).padStart(8, '0');
|
|
60
|
+
const b = timestamp.low.toString(16).padStart(8, '0');
|
|
61
|
+
|
|
62
|
+
const segments = [`${a}${b}`];
|
|
63
|
+
|
|
64
|
+
if (resumeToken) {
|
|
65
|
+
segments.push(storage.serializeBson({ resumeToken }).toString('base64'));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return segments.join(DELIMINATOR);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
toString() {
|
|
72
|
+
return this.comparable;
|
|
73
|
+
}
|
|
74
|
+
}
|