@powersync/service-core 0.0.0-dev-20241016113053 → 0.0.0-dev-20241016143203
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -8
- package/dist/db/mongo.d.ts +6 -0
- package/dist/db/mongo.js +6 -0
- package/dist/db/mongo.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +0 -6
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +1 -9
- package/dist/storage/MongoBucketStorage.js +32 -30
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +1 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.js +26 -17
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +1 -3
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js +4 -12
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/mongo/db.d.ts +9 -0
- package/dist/storage/mongo/db.js +11 -0
- package/dist/storage/mongo/db.js.map +1 -1
- package/dist/storage/write-checkpoint.d.ts +0 -1
- package/dist/storage/write-checkpoint.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +1 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/types.d.ts +0 -1
- package/package.json +5 -5
- package/src/db/mongo.ts +7 -0
- package/src/storage/BucketStorage.ts +0 -9
- package/src/storage/MongoBucketStorage.ts +32 -37
- package/src/storage/mongo/MongoSyncBucketStorage.ts +26 -21
- package/src/storage/mongo/MongoWriteCheckpointAPI.ts +8 -15
- package/src/storage/mongo/db.ts +12 -0
- package/src/storage/write-checkpoint.ts +0 -2
- package/src/util/config/compound-config-collector.ts +1 -2
- package/src/util/config/types.ts +0 -1
- package/test/src/data_storage.test.ts +22 -0
- package/test/src/sync.test.ts +2 -16
- package/test/src/util.ts +14 -3
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -75,8 +75,7 @@ export class CompoundConfigCollector {
|
|
|
75
75
|
},
|
|
76
76
|
// TODO maybe move this out of the connection or something
|
|
77
77
|
// slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
78
|
-
slot_name_prefix: 'powersync_'
|
|
79
|
-
parameters: baseConfig.parameters ?? {}
|
|
78
|
+
slot_name_prefix: 'powersync_'
|
|
80
79
|
};
|
|
81
80
|
return config;
|
|
82
81
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AA4B5F,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,eAA6B,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE;YACtD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,0DAA0D;YAC1D,qEAAqE;YACrE,gBAAgB,EAAE,YAAY;
|
|
1
|
+
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AA4B5F,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,eAA6B,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE;YACtD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,0DAA0D;YAC1D,qEAAqE;YACrE,gBAAgB,EAAE,YAAY;SAC/B,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB,CAAC,aAA2B;QAC3D,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,UAAU,EAAE,CAAC;oBACf,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,2CAA2C,SAAS,CAAC,IAAI,iDAAiD,CAC3G,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC9B,UAAsC,EACtC,YAA0B;QAE1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,qCAAqC,SAAS,CAAC,IAAI,iDAAiD,CACrG,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YAC1G,CAAC;QACH,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20241016143203",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"uuid": "^9.0.1",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
36
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20241016143203",
|
|
37
37
|
"@powersync/service-jsonbig": "0.17.10",
|
|
38
|
-
"@powersync/service-rsocket-router": "0.0.0-dev-
|
|
39
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
40
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
38
|
+
"@powersync/service-rsocket-router": "0.0.0-dev-20241016143203",
|
|
39
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20241016143203",
|
|
40
|
+
"@powersync/service-types": "0.0.0-dev-20241016143203"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/async": "^3.2.24",
|
package/src/db/mongo.ts
CHANGED
|
@@ -23,6 +23,13 @@ export const MONGO_SOCKET_TIMEOUT_MS = 60_000;
|
|
|
23
23
|
*/
|
|
24
24
|
export const MONGO_OPERATION_TIMEOUT_MS = 30_000;
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Same as above, but specifically for clear operations.
|
|
28
|
+
*
|
|
29
|
+
* These are retried when reaching the timeout.
|
|
30
|
+
*/
|
|
31
|
+
export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000;
|
|
32
|
+
|
|
26
33
|
export function createMongoClient(config: configFile.PowerSyncConfig['storage']) {
|
|
27
34
|
const normalized = normalizeMongoConfig(config);
|
|
28
35
|
return new mongo.MongoClient(normalized.uri, {
|
|
@@ -90,11 +90,6 @@ export interface BucketStorageFactory
|
|
|
90
90
|
*/
|
|
91
91
|
getActiveCheckpoint(): Promise<ActiveCheckpoint>;
|
|
92
92
|
|
|
93
|
-
/**
|
|
94
|
-
* Yields the latest sync checkpoint.
|
|
95
|
-
*/
|
|
96
|
-
watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint>;
|
|
97
|
-
|
|
98
93
|
/**
|
|
99
94
|
* Yields the latest user write checkpoint whenever the sync checkpoint updates.
|
|
100
95
|
*/
|
|
@@ -123,8 +118,6 @@ export interface ActiveCheckpoint {
|
|
|
123
118
|
hasSyncRules(): boolean;
|
|
124
119
|
|
|
125
120
|
getBucketStorage(): Promise<SyncRulesBucketStorage | null>;
|
|
126
|
-
|
|
127
|
-
syncRules: PersistedSyncRules | null;
|
|
128
121
|
}
|
|
129
122
|
|
|
130
123
|
export interface StorageMetrics {
|
|
@@ -272,8 +265,6 @@ export interface SyncRulesBucketStorage extends DisposableObserverClient<SyncRul
|
|
|
272
265
|
*/
|
|
273
266
|
clear(): Promise<void>;
|
|
274
267
|
|
|
275
|
-
setSnapshotDone(lsn: string): Promise<void>;
|
|
276
|
-
|
|
277
268
|
autoActivate(): Promise<void>;
|
|
278
269
|
|
|
279
270
|
/**
|
|
@@ -78,9 +78,6 @@ export class MongoBucketStorage
|
|
|
78
78
|
db: PowerSyncMongo,
|
|
79
79
|
options: {
|
|
80
80
|
slot_name_prefix: string;
|
|
81
|
-
/**
|
|
82
|
-
* Initial Write Checkpoint Mode
|
|
83
|
-
*/
|
|
84
81
|
write_checkpoint_mode?: WriteCheckpointMode;
|
|
85
82
|
}
|
|
86
83
|
) {
|
|
@@ -306,10 +303,6 @@ export class MongoBucketStorage
|
|
|
306
303
|
return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints);
|
|
307
304
|
}
|
|
308
305
|
|
|
309
|
-
setWriteCheckpointMode(mode: WriteCheckpointMode): void {
|
|
310
|
-
return this.writeCheckpointAPI.setWriteCheckpointMode(mode);
|
|
311
|
-
}
|
|
312
|
-
|
|
313
306
|
async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
|
|
314
307
|
return this.writeCheckpointAPI.createCustomWriteCheckpoint(options);
|
|
315
308
|
}
|
|
@@ -338,39 +331,56 @@ export class MongoBucketStorage
|
|
|
338
331
|
}
|
|
339
332
|
|
|
340
333
|
async getStorageMetrics(): Promise<StorageMetrics> {
|
|
334
|
+
const ignoreNotExiting = (e: unknown) => {
|
|
335
|
+
if (e instanceof mongo.MongoServerError && e.codeName == 'NamespaceNotFound') {
|
|
336
|
+
// Collection doesn't exist - return 0
|
|
337
|
+
return [{ storageStats: { size: 0 } }];
|
|
338
|
+
} else {
|
|
339
|
+
return Promise.reject(e);
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
|
|
343
|
+
const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
|
|
344
|
+
if (active_sync_rules == null) {
|
|
345
|
+
return {
|
|
346
|
+
operations_size_bytes: 0,
|
|
347
|
+
parameters_size_bytes: 0,
|
|
348
|
+
replication_size_bytes: 0
|
|
349
|
+
};
|
|
350
|
+
}
|
|
341
351
|
const operations_aggregate = await this.db.bucket_data
|
|
342
352
|
|
|
343
353
|
.aggregate([
|
|
344
354
|
{
|
|
345
355
|
$collStats: {
|
|
346
|
-
storageStats: {}
|
|
347
|
-
count: {}
|
|
356
|
+
storageStats: {}
|
|
348
357
|
}
|
|
349
358
|
}
|
|
350
359
|
])
|
|
351
|
-
.toArray()
|
|
360
|
+
.toArray()
|
|
361
|
+
.catch(ignoreNotExiting);
|
|
352
362
|
|
|
353
363
|
const parameters_aggregate = await this.db.bucket_parameters
|
|
354
364
|
.aggregate([
|
|
355
365
|
{
|
|
356
366
|
$collStats: {
|
|
357
|
-
storageStats: {}
|
|
358
|
-
count: {}
|
|
367
|
+
storageStats: {}
|
|
359
368
|
}
|
|
360
369
|
}
|
|
361
370
|
])
|
|
362
|
-
.toArray()
|
|
371
|
+
.toArray()
|
|
372
|
+
.catch(ignoreNotExiting);
|
|
363
373
|
|
|
364
374
|
const replication_aggregate = await this.db.current_data
|
|
365
375
|
.aggregate([
|
|
366
376
|
{
|
|
367
377
|
$collStats: {
|
|
368
|
-
storageStats: {}
|
|
369
|
-
count: {}
|
|
378
|
+
storageStats: {}
|
|
370
379
|
}
|
|
371
380
|
}
|
|
372
381
|
])
|
|
373
|
-
.toArray()
|
|
382
|
+
.toArray()
|
|
383
|
+
.catch(ignoreNotExiting);
|
|
374
384
|
|
|
375
385
|
return {
|
|
376
386
|
operations_size_bytes: operations_aggregate[0].storageStats.size,
|
|
@@ -415,19 +425,14 @@ export class MongoBucketStorage
|
|
|
415
425
|
return null;
|
|
416
426
|
}
|
|
417
427
|
return (await this.storageCache.fetch(doc._id)) ?? null;
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
? new MongoPersistedSyncRulesContent(this.db, doc).parsed({
|
|
421
|
-
defaultSchema: ''
|
|
422
|
-
})
|
|
423
|
-
: null
|
|
424
|
-
} satisfies ActiveCheckpoint;
|
|
428
|
+
}
|
|
429
|
+
};
|
|
425
430
|
}
|
|
426
431
|
|
|
427
432
|
/**
|
|
428
433
|
* Instance-wide watch on the latest available checkpoint (op_id + lsn).
|
|
429
434
|
*/
|
|
430
|
-
private async *
|
|
435
|
+
private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
|
|
431
436
|
const pipeline: mongo.Document[] = [
|
|
432
437
|
{
|
|
433
438
|
$match: {
|
|
@@ -440,8 +445,7 @@ export class MongoBucketStorage
|
|
|
440
445
|
operationType: 1,
|
|
441
446
|
'fullDocument._id': 1,
|
|
442
447
|
'fullDocument.last_checkpoint': 1,
|
|
443
|
-
'fullDocument.last_checkpoint_lsn': 1
|
|
444
|
-
'fullDocument.content': 1
|
|
448
|
+
'fullDocument.last_checkpoint_lsn': 1
|
|
445
449
|
}
|
|
446
450
|
}
|
|
447
451
|
];
|
|
@@ -463,8 +467,7 @@ export class MongoBucketStorage
|
|
|
463
467
|
projection: {
|
|
464
468
|
_id: 1,
|
|
465
469
|
last_checkpoint: 1,
|
|
466
|
-
last_checkpoint_lsn: 1
|
|
467
|
-
content: 1
|
|
470
|
+
last_checkpoint_lsn: 1
|
|
468
471
|
}
|
|
469
472
|
}
|
|
470
473
|
);
|
|
@@ -513,7 +516,6 @@ export class MongoBucketStorage
|
|
|
513
516
|
if (doc == null) {
|
|
514
517
|
continue;
|
|
515
518
|
}
|
|
516
|
-
|
|
517
519
|
const op = this.makeActiveCheckpoint(doc);
|
|
518
520
|
// Check for LSN / checkpoint changes - ignore other metadata changes
|
|
519
521
|
if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
|
|
@@ -525,16 +527,9 @@ export class MongoBucketStorage
|
|
|
525
527
|
|
|
526
528
|
// Nothing is done here until a subscriber starts to iterate
|
|
527
529
|
private readonly sharedIter = new sync.BroadcastIterable((signal) => {
|
|
528
|
-
return this.
|
|
530
|
+
return this.watchActiveCheckpoint(signal);
|
|
529
531
|
});
|
|
530
532
|
|
|
531
|
-
/**
|
|
532
|
-
* Watch changes to the active sync rules and checkpoint.
|
|
533
|
-
*/
|
|
534
|
-
watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
|
|
535
|
-
return wrapWithAbort(this.sharedIter, signal);
|
|
536
|
-
}
|
|
537
|
-
|
|
538
533
|
/**
|
|
539
534
|
* User-specific watch on the latest checkpoint and/or write checkpoint.
|
|
540
535
|
*/
|
|
@@ -31,6 +31,8 @@ import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './m
|
|
|
31
31
|
import { MongoBucketBatch } from './MongoBucketBatch.js';
|
|
32
32
|
import { MongoCompactor } from './MongoCompactor.js';
|
|
33
33
|
import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
|
|
34
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
35
|
+
import * as timers from 'timers/promises';
|
|
34
36
|
|
|
35
37
|
export class MongoSyncBucketStorage
|
|
36
38
|
extends DisposableObserver<SyncRulesBucketStorageListener>
|
|
@@ -459,10 +461,28 @@ export class MongoSyncBucketStorage
|
|
|
459
461
|
}
|
|
460
462
|
|
|
461
463
|
async clear(): Promise<void> {
|
|
464
|
+
while (true) {
|
|
465
|
+
try {
|
|
466
|
+
await this.clearIteration();
|
|
467
|
+
return;
|
|
468
|
+
} catch (e: unknown) {
|
|
469
|
+
if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') {
|
|
470
|
+
logger.info(
|
|
471
|
+
`Clearing took longer than ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, waiting and triggering another iteration.`
|
|
472
|
+
);
|
|
473
|
+
await timers.setTimeout(db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5);
|
|
474
|
+
continue;
|
|
475
|
+
} else {
|
|
476
|
+
throw e;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
private async clearIteration(): Promise<void> {
|
|
462
483
|
// Individual operations here may time out with the maxTimeMS option.
|
|
463
484
|
// It is expected to still make progress, and continue on the next try.
|
|
464
485
|
|
|
465
|
-
// TODO: Transactional?
|
|
466
486
|
await this.db.sync_rules.updateOne(
|
|
467
487
|
{
|
|
468
488
|
_id: this.group_id
|
|
@@ -476,48 +496,33 @@ export class MongoSyncBucketStorage
|
|
|
476
496
|
no_checkpoint_before: null
|
|
477
497
|
}
|
|
478
498
|
},
|
|
479
|
-
{ maxTimeMS: db.mongo.
|
|
499
|
+
{ maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
480
500
|
);
|
|
481
501
|
await this.db.bucket_data.deleteMany(
|
|
482
502
|
{
|
|
483
503
|
_id: idPrefixFilter<BucketDataKey>({ g: this.group_id }, ['b', 'o'])
|
|
484
504
|
},
|
|
485
|
-
{ maxTimeMS: db.mongo.
|
|
505
|
+
{ maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
486
506
|
);
|
|
487
507
|
await this.db.bucket_parameters.deleteMany(
|
|
488
508
|
{
|
|
489
509
|
key: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
|
|
490
510
|
},
|
|
491
|
-
{ maxTimeMS: db.mongo.
|
|
511
|
+
{ maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
492
512
|
);
|
|
493
513
|
|
|
494
514
|
await this.db.current_data.deleteMany(
|
|
495
515
|
{
|
|
496
516
|
_id: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
|
|
497
517
|
},
|
|
498
|
-
{ maxTimeMS: db.mongo.
|
|
518
|
+
{ maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
499
519
|
);
|
|
500
520
|
|
|
501
521
|
await this.db.source_tables.deleteMany(
|
|
502
522
|
{
|
|
503
523
|
group_id: this.group_id
|
|
504
524
|
},
|
|
505
|
-
{ maxTimeMS: db.mongo.
|
|
506
|
-
);
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
async setSnapshotDone(lsn: string): Promise<void> {
|
|
510
|
-
await this.db.sync_rules.updateOne(
|
|
511
|
-
{
|
|
512
|
-
_id: this.group_id
|
|
513
|
-
},
|
|
514
|
-
{
|
|
515
|
-
$set: {
|
|
516
|
-
snapshot_done: true,
|
|
517
|
-
persisted_lsn: lsn,
|
|
518
|
-
last_checkpoint_ts: new Date()
|
|
519
|
-
}
|
|
520
|
-
}
|
|
525
|
+
{ maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
521
526
|
);
|
|
522
527
|
}
|
|
523
528
|
|
|
@@ -17,19 +17,11 @@ export type MongoCheckpointAPIOptions = {
|
|
|
17
17
|
|
|
18
18
|
export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
19
19
|
readonly db: PowerSyncMongo;
|
|
20
|
-
|
|
20
|
+
readonly mode: WriteCheckpointMode;
|
|
21
21
|
|
|
22
22
|
constructor(options: MongoCheckpointAPIOptions) {
|
|
23
23
|
this.db = options.db;
|
|
24
|
-
this.
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
get mode() {
|
|
28
|
-
return this._mode;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
setWriteCheckpointMode(mode: WriteCheckpointMode): void {
|
|
32
|
-
this._mode = mode;
|
|
24
|
+
this.mode = options.mode;
|
|
33
25
|
}
|
|
34
26
|
|
|
35
27
|
async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
|
|
@@ -37,11 +29,12 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
|
|
|
37
29
|
}
|
|
38
30
|
|
|
39
31
|
async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
32
|
+
if (this.mode !== WriteCheckpointMode.CUSTOM) {
|
|
33
|
+
throw new framework.errors.ValidationError(
|
|
34
|
+
`Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
45
38
|
const { checkpoint, user_id, sync_rules_id } = options;
|
|
46
39
|
const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
|
|
47
40
|
{
|
package/src/storage/mongo/db.ts
CHANGED
|
@@ -62,6 +62,9 @@ export class PowerSyncMongo {
|
|
|
62
62
|
this.locks = this.db.collection('locks');
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Clear all collections.
|
|
67
|
+
*/
|
|
65
68
|
async clear() {
|
|
66
69
|
await this.current_data.deleteMany({});
|
|
67
70
|
await this.bucket_data.deleteMany({});
|
|
@@ -73,4 +76,13 @@ export class PowerSyncMongo {
|
|
|
73
76
|
await this.instance.deleteOne({});
|
|
74
77
|
await this.locks.deleteMany({});
|
|
75
78
|
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Drop the entire database.
|
|
82
|
+
*
|
|
83
|
+
* Primarily for tests.
|
|
84
|
+
*/
|
|
85
|
+
async drop() {
|
|
86
|
+
await this.db.dropDatabase();
|
|
87
|
+
}
|
|
76
88
|
}
|
|
@@ -55,8 +55,6 @@ export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
|
|
|
55
55
|
export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
|
|
56
56
|
|
|
57
57
|
export interface WriteCheckpointAPI {
|
|
58
|
-
setWriteCheckpointMode(mode: WriteCheckpointMode): void;
|
|
59
|
-
|
|
60
58
|
batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
|
|
61
59
|
|
|
62
60
|
createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
|
|
@@ -122,8 +122,7 @@ export class CompoundConfigCollector {
|
|
|
122
122
|
},
|
|
123
123
|
// TODO maybe move this out of the connection or something
|
|
124
124
|
// slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
125
|
-
slot_name_prefix: 'powersync_'
|
|
126
|
-
parameters: baseConfig.parameters ?? {}
|
|
125
|
+
slot_name_prefix: 'powersync_'
|
|
127
126
|
};
|
|
128
127
|
|
|
129
128
|
return config;
|
package/src/util/config/types.ts
CHANGED
|
@@ -1438,4 +1438,26 @@ bucket_definitions:
|
|
|
1438
1438
|
expect(errorCaught).true;
|
|
1439
1439
|
expect(isDisposed).true;
|
|
1440
1440
|
});
|
|
1441
|
+
|
|
1442
|
+
test('empty storage metrics', async () => {
|
|
1443
|
+
const f = await factory({ dropAll: true });
|
|
1444
|
+
|
|
1445
|
+
const metrics = await f.getStorageMetrics();
|
|
1446
|
+
expect(metrics).toEqual({
|
|
1447
|
+
operations_size_bytes: 0,
|
|
1448
|
+
parameters_size_bytes: 0,
|
|
1449
|
+
replication_size_bytes: 0
|
|
1450
|
+
});
|
|
1451
|
+
|
|
1452
|
+
const r = await f.configureSyncRules('bucket_definitions: {}');
|
|
1453
|
+
const storage = f.getInstance(r.persisted_sync_rules!);
|
|
1454
|
+
await storage.autoActivate();
|
|
1455
|
+
|
|
1456
|
+
const metrics2 = await f.getStorageMetrics();
|
|
1457
|
+
expect(metrics2).toEqual({
|
|
1458
|
+
operations_size_bytes: 0,
|
|
1459
|
+
parameters_size_bytes: 0,
|
|
1460
|
+
replication_size_bytes: 0
|
|
1461
|
+
});
|
|
1462
|
+
});
|
|
1441
1463
|
}
|
package/test/src/sync.test.ts
CHANGED
|
@@ -5,15 +5,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
|
|
|
5
5
|
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
6
6
|
import * as timers from 'timers/promises';
|
|
7
7
|
import { describe, expect, test } from 'vitest';
|
|
8
|
-
import {
|
|
9
|
-
BATCH_OPTIONS,
|
|
10
|
-
makeTestTable,
|
|
11
|
-
MONGO_STORAGE_FACTORY,
|
|
12
|
-
PARSE_OPTIONS,
|
|
13
|
-
StorageFactory,
|
|
14
|
-
ZERO_LSN
|
|
15
|
-
} from './util.js';
|
|
16
|
-
import { ParseSyncRulesOptions, StartBatchOptions } from '@/storage/BucketStorage.js';
|
|
8
|
+
import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, PARSE_OPTIONS, StorageFactory } from './util.js';
|
|
17
9
|
|
|
18
10
|
describe('sync - mongodb', function () {
|
|
19
11
|
defineTests(MONGO_STORAGE_FACTORY);
|
|
@@ -38,8 +30,7 @@ function defineTests(factory: StorageFactory) {
|
|
|
38
30
|
content: BASIC_SYNC_RULES
|
|
39
31
|
});
|
|
40
32
|
|
|
41
|
-
const storage =
|
|
42
|
-
await storage.setSnapshotDone(ZERO_LSN);
|
|
33
|
+
const storage = f.getInstance(syncRules);
|
|
43
34
|
await storage.autoActivate();
|
|
44
35
|
|
|
45
36
|
const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
@@ -91,7 +82,6 @@ function defineTests(factory: StorageFactory) {
|
|
|
91
82
|
});
|
|
92
83
|
|
|
93
84
|
const storage = await f.getInstance(syncRules);
|
|
94
|
-
await storage.setSnapshotDone(ZERO_LSN);
|
|
95
85
|
await storage.autoActivate();
|
|
96
86
|
|
|
97
87
|
const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
|
@@ -136,7 +126,6 @@ function defineTests(factory: StorageFactory) {
|
|
|
136
126
|
});
|
|
137
127
|
|
|
138
128
|
const storage = await f.getInstance(syncRules);
|
|
139
|
-
await storage.setSnapshotDone(ZERO_LSN);
|
|
140
129
|
await storage.autoActivate();
|
|
141
130
|
|
|
142
131
|
const stream = streamResponse({
|
|
@@ -164,7 +153,6 @@ function defineTests(factory: StorageFactory) {
|
|
|
164
153
|
});
|
|
165
154
|
|
|
166
155
|
const storage = await f.getInstance(syncRules);
|
|
167
|
-
await storage.setSnapshotDone(ZERO_LSN);
|
|
168
156
|
await storage.autoActivate();
|
|
169
157
|
|
|
170
158
|
const stream = streamResponse({
|
|
@@ -226,7 +214,6 @@ function defineTests(factory: StorageFactory) {
|
|
|
226
214
|
});
|
|
227
215
|
|
|
228
216
|
const storage = await f.getInstance(syncRules);
|
|
229
|
-
await storage.setSnapshotDone(ZERO_LSN);
|
|
230
217
|
await storage.autoActivate();
|
|
231
218
|
|
|
232
219
|
const exp = Date.now() / 1000 + 0.1;
|
|
@@ -265,7 +252,6 @@ function defineTests(factory: StorageFactory) {
|
|
|
265
252
|
});
|
|
266
253
|
|
|
267
254
|
const storage = await f.getInstance(syncRules);
|
|
268
|
-
await storage.setSnapshotDone(ZERO_LSN);
|
|
269
255
|
await storage.autoActivate();
|
|
270
256
|
|
|
271
257
|
await storage.startBatch(BATCH_OPTIONS, async (batch) => {
|
package/test/src/util.ts
CHANGED
|
@@ -24,11 +24,22 @@ await Metrics.initialise({
|
|
|
24
24
|
});
|
|
25
25
|
Metrics.getInstance().resetCounters();
|
|
26
26
|
|
|
27
|
-
export
|
|
27
|
+
export interface StorageOptions {
|
|
28
|
+
/**
|
|
29
|
+
* By default, collections are only cleared/
|
|
30
|
+
* Setting this to true will drop the collections completely.
|
|
31
|
+
*/
|
|
32
|
+
dropAll?: boolean;
|
|
33
|
+
}
|
|
34
|
+
export type StorageFactory = (options?: StorageOptions) => Promise<BucketStorageFactory>;
|
|
28
35
|
|
|
29
|
-
export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
|
|
36
|
+
export const MONGO_STORAGE_FACTORY: StorageFactory = async (options?: StorageOptions) => {
|
|
30
37
|
const db = await connectMongo();
|
|
31
|
-
|
|
38
|
+
if (options?.dropAll) {
|
|
39
|
+
await db.drop();
|
|
40
|
+
} else {
|
|
41
|
+
await db.clear();
|
|
42
|
+
}
|
|
32
43
|
return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
|
|
33
44
|
};
|
|
34
45
|
|