@powersync/service-module-mongodb-storage 0.0.0-dev-20250122110924 → 0.0.0-dev-20250227082606
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -10
- package/dist/migrations/MongoMigrationAgent.js +3 -0
- package/dist/migrations/MongoMigrationAgent.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +9 -20
- package/dist/storage/MongoBucketStorage.js +86 -199
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +3 -3
- package/dist/storage/implementation/MongoBucketBatch.js +37 -24
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +0 -6
- package/dist/storage/implementation/MongoCompactor.js +11 -4
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.js +1 -0
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.js +4 -0
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +9 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +3 -1
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +16 -3
- package/dist/storage/implementation/MongoSyncBucketStorage.js +225 -22
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +5 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +3 -2
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/OperationBatch.js +10 -6
- package/dist/storage/implementation/OperationBatch.js.map +1 -1
- package/dist/storage/implementation/PersistedBatch.js +18 -18
- package/dist/storage/implementation/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +1 -1
- package/dist/storage/implementation/db.js +15 -3
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +1 -0
- package/dist/storage/implementation/util.d.ts +0 -14
- package/dist/storage/implementation/util.js +3 -41
- package/dist/storage/implementation/util.js.map +1 -1
- package/package.json +8 -8
- package/src/storage/MongoBucketStorage.ts +88 -232
- package/src/storage/implementation/MongoBucketBatch.ts +12 -10
- package/src/storage/implementation/MongoCompactor.ts +2 -10
- package/src/storage/implementation/MongoStorageProvider.ts +3 -1
- package/src/storage/implementation/MongoSyncBucketStorage.ts +292 -37
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +1 -3
- package/src/storage/implementation/PersistedBatch.ts +4 -5
- package/src/storage/implementation/db.ts +3 -3
- package/src/storage/implementation/models.ts +5 -0
- package/src/storage/implementation/util.ts +0 -45
- package/test/src/__snapshots__/storage_sync.test.ts.snap +138 -0
- package/test/src/storage_compacting.test.ts +1 -7
- package/test/src/storage_sync.test.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,iBAAiB,EAAE,
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAgBf;IAfV,UAAU,GAAsD,EAAE,CAAC;IACnE,gBAAgB,GAA2D,EAAE,CAAC;IAC9E,WAAW,GAAuD,EAAE,CAAC;IAErE;;OAEG;IACH,aAAa,GAAkB,IAAI,CAAC;IAEpC;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACU,QAAgB,EACxB,WAAmB;QADX,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE3F,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC9C,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAC1D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE;gBAChD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,QAAQ,YAAY,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC5F,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,aAAa,EAAE,CACxF,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;CACF"}
|
|
@@ -33,4 +33,4 @@ export declare class PowerSyncMongo {
|
|
|
33
33
|
*/
|
|
34
34
|
drop(): Promise<void>;
|
|
35
35
|
}
|
|
36
|
-
export declare function createPowerSyncMongo(config: MongoStorageConfig): PowerSyncMongo;
|
|
36
|
+
export declare function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions): PowerSyncMongo;
|
|
@@ -1,10 +1,22 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { storage } from '@powersync/service-core';
|
|
3
3
|
export class PowerSyncMongo {
|
|
4
|
+
current_data;
|
|
5
|
+
bucket_data;
|
|
6
|
+
bucket_parameters;
|
|
7
|
+
op_id_sequence;
|
|
8
|
+
sync_rules;
|
|
9
|
+
source_tables;
|
|
10
|
+
custom_write_checkpoints;
|
|
11
|
+
write_checkpoints;
|
|
12
|
+
instance;
|
|
13
|
+
locks;
|
|
14
|
+
client;
|
|
15
|
+
db;
|
|
4
16
|
constructor(client, options) {
|
|
5
17
|
this.client = client;
|
|
6
18
|
const db = client.db(options?.database, {
|
|
7
|
-
...storage.
|
|
19
|
+
...storage.BSON_DESERIALIZE_INTERNAL_OPTIONS
|
|
8
20
|
});
|
|
9
21
|
this.db = db;
|
|
10
22
|
this.current_data = db.collection('current_data');
|
|
@@ -41,7 +53,7 @@ export class PowerSyncMongo {
|
|
|
41
53
|
await this.db.dropDatabase();
|
|
42
54
|
}
|
|
43
55
|
}
|
|
44
|
-
export function createPowerSyncMongo(config) {
|
|
45
|
-
return new PowerSyncMongo(lib_mongo.createMongoClient(config), { database: config.database });
|
|
56
|
+
export function createPowerSyncMongo(config, options) {
|
|
57
|
+
return new PowerSyncMongo(lib_mongo.createMongoClient(config, options), { database: config.database });
|
|
46
58
|
}
|
|
47
59
|
//# sourceMappingURL=db.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAsBlD,MAAM,OAAO,cAAc;
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAsBlD,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAE9C,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AACzG,CAAC"}
|
|
@@ -122,6 +122,7 @@ export interface SyncRuleDocument {
|
|
|
122
122
|
last_fatal_error: string | null;
|
|
123
123
|
content: string;
|
|
124
124
|
}
|
|
125
|
+
export type SyncRuleCheckpointState = Pick<SyncRuleDocument, 'last_checkpoint' | 'last_checkpoint_lsn' | '_id' | 'state'>;
|
|
125
126
|
export interface CustomWriteCheckpointDocument {
|
|
126
127
|
_id: bson.ObjectId;
|
|
127
128
|
user_id: string;
|
|
@@ -30,17 +30,3 @@ export declare const createMongoClient: (url: string, options?: mongo.MongoClien
|
|
|
30
30
|
* Helper for unit tests
|
|
31
31
|
*/
|
|
32
32
|
export declare const connectMongoForTests: (url: string, isCI: boolean) => PowerSyncMongo;
|
|
33
|
-
/**
|
|
34
|
-
* MongoDB bulkWrite internally splits the operations into batches
|
|
35
|
-
* so that no batch exceeds 16MB. However, there are cases where
|
|
36
|
-
* the batch size is very close to 16MB, where additional metadata
|
|
37
|
-
* on the server pushes it over the limit, resulting in this error
|
|
38
|
-
* from the server:
|
|
39
|
-
*
|
|
40
|
-
* > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
|
|
41
|
-
*
|
|
42
|
-
* We work around the issue by doing our own batching, limiting the
|
|
43
|
-
* batch size to 15MB. This does add additional overhead with
|
|
44
|
-
* BSON.calculateObjectSize.
|
|
45
|
-
*/
|
|
46
|
-
export declare function safeBulkWrite<T extends mongo.Document>(collection: mongo.Collection<T>, operations: mongo.AnyBulkWriteOperation<T>[], options: mongo.BulkWriteOptions): Promise<void>;
|
|
@@ -108,48 +108,10 @@ export const connectMongoForTests = (url, isCI) => {
|
|
|
108
108
|
// Short timeout for tests, to fail fast when the server is not available.
|
|
109
109
|
// Slightly longer timeouts for CI, to avoid arbitrary test failures
|
|
110
110
|
const client = createMongoClient(url, {
|
|
111
|
-
connectTimeoutMS: isCI ?
|
|
112
|
-
socketTimeoutMS: isCI ?
|
|
113
|
-
serverSelectionTimeoutMS: isCI ?
|
|
111
|
+
connectTimeoutMS: isCI ? 15_000 : 5_000,
|
|
112
|
+
socketTimeoutMS: isCI ? 15_000 : 5_000,
|
|
113
|
+
serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
|
|
114
114
|
});
|
|
115
115
|
return new PowerSyncMongo(client);
|
|
116
116
|
};
|
|
117
|
-
/**
|
|
118
|
-
* MongoDB bulkWrite internally splits the operations into batches
|
|
119
|
-
* so that no batch exceeds 16MB. However, there are cases where
|
|
120
|
-
* the batch size is very close to 16MB, where additional metadata
|
|
121
|
-
* on the server pushes it over the limit, resulting in this error
|
|
122
|
-
* from the server:
|
|
123
|
-
*
|
|
124
|
-
* > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
|
|
125
|
-
*
|
|
126
|
-
* We work around the issue by doing our own batching, limiting the
|
|
127
|
-
* batch size to 15MB. This does add additional overhead with
|
|
128
|
-
* BSON.calculateObjectSize.
|
|
129
|
-
*/
|
|
130
|
-
export async function safeBulkWrite(collection, operations, options) {
|
|
131
|
-
// Must be below 16MB.
|
|
132
|
-
// We could probably go a little closer, but 15MB is a safe threshold.
|
|
133
|
-
const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
|
|
134
|
-
let batch = [];
|
|
135
|
-
let currentSize = 0;
|
|
136
|
-
// Estimated overhead per operation, should be smaller in reality.
|
|
137
|
-
const keySize = 8;
|
|
138
|
-
for (let op of operations) {
|
|
139
|
-
const bsonSize = mongo.BSON.calculateObjectSize(op, {
|
|
140
|
-
checkKeys: false,
|
|
141
|
-
ignoreUndefined: true
|
|
142
|
-
}) + keySize;
|
|
143
|
-
if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
|
|
144
|
-
await collection.bulkWrite(batch, options);
|
|
145
|
-
currentSize = 0;
|
|
146
|
-
batch = [];
|
|
147
|
-
}
|
|
148
|
-
batch.push(op);
|
|
149
|
-
currentSize += bsonSize;
|
|
150
|
-
}
|
|
151
|
-
if (batch.length > 0) {
|
|
152
|
-
await collection.bulkWrite(batch, options);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
117
|
//# sourceMappingURL=util.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.0.0-dev-
|
|
5
|
+
"version": "0.0.0-dev-20250227082606",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-Apache-2.0",
|
|
8
8
|
"type": "module",
|
|
@@ -22,21 +22,21 @@
|
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"bson": "^6.
|
|
25
|
+
"bson": "^6.10.3",
|
|
26
26
|
"ts-codec": "^1.3.0",
|
|
27
27
|
"ix": "^5.0.0",
|
|
28
28
|
"lru-cache": "^10.2.2",
|
|
29
29
|
"uuid": "^9.0.1",
|
|
30
|
-
"@powersync/lib-services-framework": "0.
|
|
31
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
30
|
+
"@powersync/lib-services-framework": "0.5.3",
|
|
31
|
+
"@powersync/service-core": "0.0.0-dev-20250227082606",
|
|
32
32
|
"@powersync/service-jsonbig": "0.17.10",
|
|
33
|
-
"@powersync/service-sync-rules": "0.
|
|
34
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
35
|
-
"@powersync/lib-service-mongodb": "0.0.0-dev-
|
|
33
|
+
"@powersync/service-sync-rules": "0.24.0",
|
|
34
|
+
"@powersync/service-types": "0.0.0-dev-20250227082606",
|
|
35
|
+
"@powersync/lib-service-mongodb": "0.0.0-dev-20250227082606"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"@types/uuid": "^9.0.4",
|
|
39
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
39
|
+
"@powersync/service-core-tests": "0.0.0-dev-20250227082606"
|
|
40
40
|
},
|
|
41
41
|
"scripts": {
|
|
42
42
|
"build": "tsc -b",
|
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
2
|
-
import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
|
|
3
|
-
import { LRUCache } from 'lru-cache/min';
|
|
4
|
-
import * as timers from 'timers/promises';
|
|
5
2
|
|
|
6
|
-
import {
|
|
3
|
+
import { GetIntanceOptions, storage } from '@powersync/service-core';
|
|
7
4
|
|
|
8
|
-
import {
|
|
5
|
+
import { BaseObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
|
|
9
6
|
import { v4 as uuid } from 'uuid';
|
|
10
7
|
|
|
11
8
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
@@ -18,7 +15,7 @@ import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.
|
|
|
18
15
|
import { generateSlotName } from './implementation/util.js';
|
|
19
16
|
|
|
20
17
|
export class MongoBucketStorage
|
|
21
|
-
extends
|
|
18
|
+
extends BaseObserver<storage.BucketStorageFactoryListener>
|
|
22
19
|
implements storage.BucketStorageFactory
|
|
23
20
|
{
|
|
24
21
|
private readonly client: mongo.MongoClient;
|
|
@@ -26,26 +23,7 @@ export class MongoBucketStorage
|
|
|
26
23
|
// TODO: This is still Postgres specific and needs to be reworked
|
|
27
24
|
public readonly slot_name_prefix: string;
|
|
28
25
|
|
|
29
|
-
private
|
|
30
|
-
max: 3,
|
|
31
|
-
fetchMethod: async (id) => {
|
|
32
|
-
const doc2 = await this.db.sync_rules.findOne(
|
|
33
|
-
{
|
|
34
|
-
_id: id
|
|
35
|
-
},
|
|
36
|
-
{ limit: 1 }
|
|
37
|
-
);
|
|
38
|
-
if (doc2 == null) {
|
|
39
|
-
// Deleted in the meantime?
|
|
40
|
-
return undefined;
|
|
41
|
-
}
|
|
42
|
-
const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
|
|
43
|
-
return this.getInstance(rules);
|
|
44
|
-
},
|
|
45
|
-
dispose: (storage) => {
|
|
46
|
-
storage[Symbol.dispose]();
|
|
47
|
-
}
|
|
48
|
-
});
|
|
26
|
+
private activeStorageCache: MongoSyncBucketStorage | undefined;
|
|
49
27
|
|
|
50
28
|
public readonly db: PowerSyncMongo;
|
|
51
29
|
|
|
@@ -63,20 +41,21 @@ export class MongoBucketStorage
|
|
|
63
41
|
}
|
|
64
42
|
|
|
65
43
|
async [Symbol.asyncDispose]() {
|
|
66
|
-
|
|
44
|
+
// No-op
|
|
67
45
|
}
|
|
68
46
|
|
|
69
|
-
getInstance(
|
|
70
|
-
let { id, slot_name } =
|
|
47
|
+
getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage {
|
|
48
|
+
let { id, slot_name } = syncRules;
|
|
71
49
|
if ((typeof id as any) == 'bigint') {
|
|
72
50
|
id = Number(id);
|
|
73
51
|
}
|
|
74
|
-
const storage = new MongoSyncBucketStorage(this, id,
|
|
75
|
-
|
|
52
|
+
const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name);
|
|
53
|
+
if (!options?.skipLifecycleHooks) {
|
|
54
|
+
this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
|
|
55
|
+
}
|
|
76
56
|
storage.registerListener({
|
|
77
57
|
batchStarted: (batch) => {
|
|
78
|
-
|
|
79
|
-
batch.registerManagedListener(storage, {
|
|
58
|
+
batch.registerListener({
|
|
80
59
|
replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
|
|
81
60
|
});
|
|
82
61
|
}
|
|
@@ -84,36 +63,49 @@ export class MongoBucketStorage
|
|
|
84
63
|
return storage;
|
|
85
64
|
}
|
|
86
65
|
|
|
87
|
-
async
|
|
66
|
+
async getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier> {
|
|
67
|
+
const { setName: id } = await this.db.db.command({
|
|
68
|
+
hello: 1
|
|
69
|
+
});
|
|
70
|
+
if (id == null) {
|
|
71
|
+
throw new ServiceError(
|
|
72
|
+
ErrorCode.PSYNC_S1342,
|
|
73
|
+
'Standalone MongoDB instances are not supported - use a replicaset.'
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
id,
|
|
79
|
+
type: lib_mongo.MONGO_CONNECTION_TYPE
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async configureSyncRules(options: storage.UpdateSyncRulesOptions) {
|
|
88
84
|
const next = await this.getNextSyncRulesContent();
|
|
89
85
|
const active = await this.getActiveSyncRulesContent();
|
|
90
86
|
|
|
91
|
-
if (next?.sync_rules_content ==
|
|
87
|
+
if (next?.sync_rules_content == options.content) {
|
|
92
88
|
logger.info('Sync rules from configuration unchanged');
|
|
93
89
|
return { updated: false };
|
|
94
|
-
} else if (next == null && active?.sync_rules_content ==
|
|
90
|
+
} else if (next == null && active?.sync_rules_content == options.content) {
|
|
95
91
|
logger.info('Sync rules from configuration unchanged');
|
|
96
92
|
return { updated: false };
|
|
97
93
|
} else {
|
|
98
94
|
logger.info('Sync rules updated from configuration');
|
|
99
|
-
const persisted_sync_rules = await this.updateSyncRules(
|
|
100
|
-
content: sync_rules,
|
|
101
|
-
lock: options?.lock
|
|
102
|
-
});
|
|
95
|
+
const persisted_sync_rules = await this.updateSyncRules(options);
|
|
103
96
|
return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
|
|
104
97
|
}
|
|
105
98
|
}
|
|
106
99
|
|
|
107
|
-
async
|
|
100
|
+
async restartReplication(sync_rules_group_id: number) {
|
|
108
101
|
const next = await this.getNextSyncRulesContent();
|
|
109
102
|
const active = await this.getActiveSyncRulesContent();
|
|
110
103
|
|
|
111
|
-
|
|
112
|
-
// The current one will continue erroring until the next one has finished processing.
|
|
113
|
-
if (next != null && next.slot_name == slot_name) {
|
|
104
|
+
if (next != null && next.id == sync_rules_group_id) {
|
|
114
105
|
// We need to redo the "next" sync rules
|
|
115
106
|
await this.updateSyncRules({
|
|
116
|
-
content: next.sync_rules_content
|
|
107
|
+
content: next.sync_rules_content,
|
|
108
|
+
validate: false
|
|
117
109
|
});
|
|
118
110
|
// Pro-actively stop replicating
|
|
119
111
|
await this.db.sync_rules.updateOne(
|
|
@@ -127,13 +119,17 @@ export class MongoBucketStorage
|
|
|
127
119
|
}
|
|
128
120
|
}
|
|
129
121
|
);
|
|
130
|
-
} else if (next == null && active?.
|
|
122
|
+
} else if (next == null && active?.id == sync_rules_group_id) {
|
|
131
123
|
// Slot removed for "active" sync rules, while there is no "next" one.
|
|
132
124
|
await this.updateSyncRules({
|
|
133
|
-
content: active.sync_rules_content
|
|
125
|
+
content: active.sync_rules_content,
|
|
126
|
+
validate: false
|
|
134
127
|
});
|
|
135
128
|
|
|
136
|
-
//
|
|
129
|
+
// In this case we keep the old one as active for clients, so that that existing clients
|
|
130
|
+
// can still get the latest data while we replicate the new ones.
|
|
131
|
+
// It will however not replicate anymore.
|
|
132
|
+
|
|
137
133
|
await this.db.sync_rules.updateOne(
|
|
138
134
|
{
|
|
139
135
|
_id: active.id,
|
|
@@ -141,7 +137,21 @@ export class MongoBucketStorage
|
|
|
141
137
|
},
|
|
142
138
|
{
|
|
143
139
|
$set: {
|
|
144
|
-
state: storage.SyncRuleState.
|
|
140
|
+
state: storage.SyncRuleState.ERRORED
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
);
|
|
144
|
+
} else if (next != null && active?.id == sync_rules_group_id) {
|
|
145
|
+
// Already have next sync rules, but need to stop replicating the active one.
|
|
146
|
+
|
|
147
|
+
await this.db.sync_rules.updateOne(
|
|
148
|
+
{
|
|
149
|
+
_id: active.id,
|
|
150
|
+
state: storage.SyncRuleState.ACTIVE
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
$set: {
|
|
154
|
+
state: storage.SyncRuleState.ERRORED
|
|
145
155
|
}
|
|
146
156
|
}
|
|
147
157
|
);
|
|
@@ -149,13 +159,18 @@ export class MongoBucketStorage
|
|
|
149
159
|
}
|
|
150
160
|
|
|
151
161
|
async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
162
|
+
if (options.validate) {
|
|
163
|
+
// Parse and validate before applying any changes
|
|
164
|
+
SqlSyncRules.fromYaml(options.content, {
|
|
165
|
+
// No schema-based validation at this point
|
|
166
|
+
schema: undefined,
|
|
167
|
+
defaultSchema: 'not_applicable', // Not needed for validation
|
|
168
|
+
throwOnError: true
|
|
169
|
+
});
|
|
170
|
+
} else {
|
|
171
|
+
// We do not validate sync rules at this point.
|
|
172
|
+
// That is done when using the sync rules, so that the diagnostics API can report the errors.
|
|
173
|
+
}
|
|
159
174
|
|
|
160
175
|
let rules: MongoPersistedSyncRulesContent | undefined = undefined;
|
|
161
176
|
|
|
@@ -213,7 +228,7 @@ export class MongoBucketStorage
|
|
|
213
228
|
async getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
|
|
214
229
|
const doc = await this.db.sync_rules.findOne(
|
|
215
230
|
{
|
|
216
|
-
state: storage.SyncRuleState.ACTIVE
|
|
231
|
+
state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
|
|
217
232
|
},
|
|
218
233
|
{ sort: { _id: -1 }, limit: 1 }
|
|
219
234
|
);
|
|
@@ -251,7 +266,7 @@ export class MongoBucketStorage
|
|
|
251
266
|
async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
|
|
252
267
|
const docs = await this.db.sync_rules
|
|
253
268
|
.find({
|
|
254
|
-
|
|
269
|
+
state: { $in: [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE] }
|
|
255
270
|
})
|
|
256
271
|
.toArray();
|
|
257
272
|
|
|
@@ -272,19 +287,22 @@ export class MongoBucketStorage
|
|
|
272
287
|
});
|
|
273
288
|
}
|
|
274
289
|
|
|
275
|
-
async
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
{
|
|
281
|
-
sort: { _id: -1 },
|
|
282
|
-
limit: 1,
|
|
283
|
-
projection: { _id: 1, last_checkpoint: 1, last_checkpoint_lsn: 1 }
|
|
284
|
-
}
|
|
285
|
-
);
|
|
290
|
+
async getActiveStorage(): Promise<MongoSyncBucketStorage | null> {
|
|
291
|
+
const content = await this.getActiveSyncRulesContent();
|
|
292
|
+
if (content == null) {
|
|
293
|
+
return null;
|
|
294
|
+
}
|
|
286
295
|
|
|
287
|
-
|
|
296
|
+
// It is important that this instance is cached.
|
|
297
|
+
// Not for the instance construction itself, but to ensure that internal caches on the instance
|
|
298
|
+
// are re-used properly.
|
|
299
|
+
if (this.activeStorageCache?.group_id == content.id) {
|
|
300
|
+
return this.activeStorageCache;
|
|
301
|
+
} else {
|
|
302
|
+
const instance = this.getInstance(content);
|
|
303
|
+
this.activeStorageCache = instance;
|
|
304
|
+
return instance;
|
|
305
|
+
}
|
|
288
306
|
}
|
|
289
307
|
|
|
290
308
|
async getStorageMetrics(): Promise<storage.StorageMetrics> {
|
|
@@ -370,166 +388,4 @@ export class MongoBucketStorage
|
|
|
370
388
|
|
|
371
389
|
return instance!._id;
|
|
372
390
|
}
|
|
373
|
-
|
|
374
|
-
private makeActiveCheckpoint(doc: SyncRuleDocument | null) {
|
|
375
|
-
return {
|
|
376
|
-
checkpoint: utils.timestampToOpId(doc?.last_checkpoint ?? 0n),
|
|
377
|
-
lsn: doc?.last_checkpoint_lsn ?? null,
|
|
378
|
-
hasSyncRules() {
|
|
379
|
-
return doc != null;
|
|
380
|
-
},
|
|
381
|
-
getBucketStorage: async () => {
|
|
382
|
-
if (doc == null) {
|
|
383
|
-
return null;
|
|
384
|
-
}
|
|
385
|
-
return (await this.storageCache.fetch(doc._id)) ?? null;
|
|
386
|
-
}
|
|
387
|
-
} satisfies storage.ActiveCheckpoint;
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
/**
|
|
391
|
-
* Instance-wide watch on the latest available checkpoint (op_id + lsn).
|
|
392
|
-
*/
|
|
393
|
-
private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<storage.ActiveCheckpoint> {
|
|
394
|
-
const pipeline: mongo.Document[] = [
|
|
395
|
-
{
|
|
396
|
-
$match: {
|
|
397
|
-
'fullDocument.state': 'ACTIVE',
|
|
398
|
-
operationType: { $in: ['insert', 'update'] }
|
|
399
|
-
}
|
|
400
|
-
},
|
|
401
|
-
{
|
|
402
|
-
$project: {
|
|
403
|
-
operationType: 1,
|
|
404
|
-
'fullDocument._id': 1,
|
|
405
|
-
'fullDocument.last_checkpoint': 1,
|
|
406
|
-
'fullDocument.last_checkpoint_lsn': 1
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
];
|
|
410
|
-
|
|
411
|
-
// Use this form instead of (doc: SyncRuleDocument | null = null),
|
|
412
|
-
// otherwise we get weird "doc: never" issues.
|
|
413
|
-
let doc = null as SyncRuleDocument | null;
|
|
414
|
-
let clusterTime = null as mongo.Timestamp | null;
|
|
415
|
-
|
|
416
|
-
await this.client.withSession(async (session) => {
|
|
417
|
-
doc = await this.db.sync_rules.findOne(
|
|
418
|
-
{
|
|
419
|
-
state: storage.SyncRuleState.ACTIVE
|
|
420
|
-
},
|
|
421
|
-
{
|
|
422
|
-
session,
|
|
423
|
-
sort: { _id: -1 },
|
|
424
|
-
limit: 1,
|
|
425
|
-
projection: {
|
|
426
|
-
_id: 1,
|
|
427
|
-
last_checkpoint: 1,
|
|
428
|
-
last_checkpoint_lsn: 1
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
);
|
|
432
|
-
const time = session.clusterTime?.clusterTime ?? null;
|
|
433
|
-
clusterTime = time;
|
|
434
|
-
});
|
|
435
|
-
if (clusterTime == null) {
|
|
436
|
-
throw new ServiceError(ErrorCode.PSYNC_S2401, 'Could not get clusterTime');
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
if (signal.aborted) {
|
|
440
|
-
return;
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
if (doc) {
|
|
444
|
-
yield this.makeActiveCheckpoint(doc);
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
const stream = this.db.sync_rules.watch(pipeline, {
|
|
448
|
-
fullDocument: 'updateLookup',
|
|
449
|
-
// Start at the cluster time where we got the initial doc, to make sure
|
|
450
|
-
// we don't skip any updates.
|
|
451
|
-
// This may result in the first operation being a duplicate, but we filter
|
|
452
|
-
// it out anyway.
|
|
453
|
-
startAtOperationTime: clusterTime
|
|
454
|
-
});
|
|
455
|
-
|
|
456
|
-
signal.addEventListener(
|
|
457
|
-
'abort',
|
|
458
|
-
() => {
|
|
459
|
-
stream.close();
|
|
460
|
-
},
|
|
461
|
-
{ once: true }
|
|
462
|
-
);
|
|
463
|
-
|
|
464
|
-
let lastOp: storage.ActiveCheckpoint | null = null;
|
|
465
|
-
|
|
466
|
-
for await (const update of stream.stream()) {
|
|
467
|
-
if (signal.aborted) {
|
|
468
|
-
break;
|
|
469
|
-
}
|
|
470
|
-
if (update.operationType != 'insert' && update.operationType != 'update') {
|
|
471
|
-
continue;
|
|
472
|
-
}
|
|
473
|
-
const doc = update.fullDocument!;
|
|
474
|
-
if (doc == null) {
|
|
475
|
-
continue;
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
const op = this.makeActiveCheckpoint(doc);
|
|
479
|
-
// Check for LSN / checkpoint changes - ignore other metadata changes
|
|
480
|
-
if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
|
|
481
|
-
lastOp = op;
|
|
482
|
-
yield op;
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
// Nothing is done here until a subscriber starts to iterate
|
|
488
|
-
private readonly sharedIter = new sync.BroadcastIterable((signal) => {
|
|
489
|
-
return this.watchActiveCheckpoint(signal);
|
|
490
|
-
});
|
|
491
|
-
|
|
492
|
-
/**
|
|
493
|
-
* User-specific watch on the latest checkpoint and/or write checkpoint.
|
|
494
|
-
*/
|
|
495
|
-
async *watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable<storage.WriteCheckpoint> {
|
|
496
|
-
let lastCheckpoint: utils.OpId | null = null;
|
|
497
|
-
let lastWriteCheckpoint: bigint | null = null;
|
|
498
|
-
|
|
499
|
-
const iter = wrapWithAbort(this.sharedIter, signal);
|
|
500
|
-
for await (const cp of iter) {
|
|
501
|
-
const { checkpoint, lsn } = cp;
|
|
502
|
-
|
|
503
|
-
// lsn changes are not important by itself.
|
|
504
|
-
// What is important is:
|
|
505
|
-
// 1. checkpoint (op_id) changes.
|
|
506
|
-
// 2. write checkpoint changes for the specific user
|
|
507
|
-
const bucketStorage = await cp.getBucketStorage();
|
|
508
|
-
if (!bucketStorage) {
|
|
509
|
-
continue;
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
|
|
513
|
-
|
|
514
|
-
const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({
|
|
515
|
-
user_id,
|
|
516
|
-
heads: {
|
|
517
|
-
...lsnFilters
|
|
518
|
-
}
|
|
519
|
-
});
|
|
520
|
-
|
|
521
|
-
if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
|
|
522
|
-
// No change - wait for next one
|
|
523
|
-
// In some cases, many LSNs may be produced in a short time.
|
|
524
|
-
// Add a delay to throttle the write checkpoint lookup a bit.
|
|
525
|
-
await timers.setTimeout(20 + 10 * Math.random());
|
|
526
|
-
continue;
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
lastWriteCheckpoint = currentWriteCheckpoint;
|
|
530
|
-
lastCheckpoint = checkpoint;
|
|
531
|
-
|
|
532
|
-
yield { base: cp, writeCheckpoint: currentWriteCheckpoint };
|
|
533
|
-
}
|
|
534
|
-
}
|
|
535
391
|
}
|