@powersync/service-core 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/metrics/Metrics.js.map +1 -1
- package/dist/replication/ErrorRateLimiter.js +2 -1
- package/dist/replication/ErrorRateLimiter.js.map +1 -1
- package/dist/replication/WalStream.js +7 -5
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/routes/router.d.ts +1 -0
- package/dist/routes/socket-route.js +1 -1
- package/dist/routes/socket-route.js.map +1 -1
- package/dist/storage/MongoBucketStorage.js +1 -1
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/mongo/MongoBucketBatch.js +2 -5
- package/dist/storage/mongo/MongoBucketBatch.js.map +1 -1
- package/dist/storage/mongo/OperationBatch.js +1 -1
- package/dist/storage/mongo/OperationBatch.js.map +1 -1
- package/dist/storage/mongo/PersistedBatch.d.ts +4 -0
- package/dist/storage/mongo/PersistedBatch.js +15 -2
- package/dist/storage/mongo/PersistedBatch.js.map +1 -1
- package/dist/storage/mongo/db.js +0 -3
- package/dist/storage/mongo/db.js.map +1 -1
- package/dist/storage/mongo/util.js +1 -1
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/package.json +6 -6
- package/src/metrics/Metrics.ts +5 -3
- package/src/replication/ErrorRateLimiter.ts +2 -1
- package/src/replication/WalStream.ts +8 -5
- package/src/routes/router.ts +1 -0
- package/src/routes/socket-route.ts +1 -1
- package/src/storage/MongoBucketStorage.ts +2 -2
- package/src/storage/mongo/MongoBucketBatch.ts +2 -4
- package/src/storage/mongo/OperationBatch.ts +1 -1
- package/src/storage/mongo/PersistedBatch.ts +25 -2
- package/src/storage/mongo/db.ts +0 -3
- package/src/storage/mongo/util.ts +1 -1
- package/src/util/config/compound-config-collector.ts +2 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAc,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG3E,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AAEzF,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAiB1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,gBAA8B,EAAE;QAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACnC,MAAM,IAAI,GAAuB;gBAC/B,IAAI,EAAE,YAAqB;gBAC3B,GAAG,mBAAmB,CAAC,CAAC,CAAC;gBACzB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,KAAK;aAChC,CAAC;YAEF,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAE1E,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,
|
|
1
|
+
{"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAc,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG3E,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AAEzF,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAiB1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,gBAA8B,EAAE;QAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACnC,MAAM,IAAI,GAAuB;gBAC/B,IAAI,EAAE,YAAqB;gBAC3B,GAAG,mBAAmB,CAAC,CAAC,CAAC;gBACzB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,KAAK;aAChC,CAAC;YAEF,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAE1E,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,gBAAgB,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,gBAAgB,IAAI,YAAY;SACnE,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB,CAAC,aAA2B;QAC3D,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,UAAU,EAAE,CAAC;oBACf,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAChB,2CAA2C,SAAS,CAAC,IAAI,iDAAiD,CAC3G,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC9B,UAAsC,EACtC,YAA0B;QAE1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAChB,qCAAqC,SAAS,CAAC,IAAI,iDAAiD,CACrG,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YAC1G,CAAC;QACH,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.1.
|
|
8
|
+
"version": "0.1.3",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -26,17 +26,17 @@
|
|
|
26
26
|
"ix": "^5.0.0",
|
|
27
27
|
"jose": "^4.15.1",
|
|
28
28
|
"lru-cache": "^10.2.2",
|
|
29
|
-
"mongodb": "^6.
|
|
29
|
+
"mongodb": "^6.7.0",
|
|
30
30
|
"node-fetch": "^3.3.2",
|
|
31
31
|
"pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
|
|
32
32
|
"ts-codec": "^1.2.2",
|
|
33
33
|
"uuid": "^9.0.1",
|
|
34
34
|
"yaml": "^2.3.2",
|
|
35
|
-
"@powersync/service-jpgwire": "0.17.
|
|
36
|
-
"@powersync/service-
|
|
37
|
-
"@powersync/service-
|
|
35
|
+
"@powersync/service-jpgwire": "0.17.12",
|
|
36
|
+
"@powersync/service-jsonbig": "0.17.10",
|
|
37
|
+
"@powersync/service-rsocket-router": "0.0.7",
|
|
38
38
|
"@powersync/service-types": "0.1.0",
|
|
39
|
-
"@powersync/service-
|
|
39
|
+
"@powersync/service-sync-rules": "0.17.10"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@journeyapps-platform/micro-dev": "^1.6.7",
|
package/src/metrics/Metrics.ts
CHANGED
|
@@ -146,14 +146,16 @@ export class Metrics {
|
|
|
146
146
|
}
|
|
147
147
|
micro.logger.info('Configuring telemetry.');
|
|
148
148
|
|
|
149
|
-
micro.logger.info(
|
|
149
|
+
micro.logger.info(
|
|
150
|
+
`
|
|
150
151
|
Attention:
|
|
151
152
|
PowerSync collects completely anonymous telemetry regarding usage.
|
|
152
153
|
This information is used to shape our roadmap to better serve our customers.
|
|
153
154
|
You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL:
|
|
154
155
|
https://docs.powersync.com/self-hosting/telemetry
|
|
155
156
|
Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'}
|
|
156
|
-
`.trim()
|
|
157
|
+
`.trim()
|
|
158
|
+
);
|
|
157
159
|
|
|
158
160
|
const configuredExporters: MetricReader[] = [];
|
|
159
161
|
|
|
@@ -167,7 +169,7 @@ Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disable
|
|
|
167
169
|
exporter: new OTLPMetricExporter({
|
|
168
170
|
url: options.internal_metrics_endpoint
|
|
169
171
|
}),
|
|
170
|
-
exportIntervalMillis: 1000 * 60 * 5
|
|
172
|
+
exportIntervalMillis: 1000 * 60 * 5 // 5 minutes
|
|
171
173
|
});
|
|
172
174
|
|
|
173
175
|
configuredExporters.push(periodicExporter);
|
|
@@ -12,7 +12,8 @@ export class DefaultErrorRateLimiter implements ErrorRateLimiter {
|
|
|
12
12
|
|
|
13
13
|
async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
|
|
14
14
|
const delay = Math.max(0, this.nextAllowed - Date.now());
|
|
15
|
-
|
|
15
|
+
// Minimum delay between connections, even without errors
|
|
16
|
+
this.setDelay(500);
|
|
16
17
|
await setTimeout(delay, undefined, { signal: options?.signal });
|
|
17
18
|
}
|
|
18
19
|
|
|
@@ -274,7 +274,7 @@ WHERE oid = $1::regclass`,
|
|
|
274
274
|
params: [{ value: table.qualifiedName, type: 'varchar' }]
|
|
275
275
|
});
|
|
276
276
|
const row = results.rows[0];
|
|
277
|
-
if (row?.[0] ?? -1n == -1n) {
|
|
277
|
+
if ((row?.[0] ?? -1n) == -1n) {
|
|
278
278
|
return '?';
|
|
279
279
|
} else {
|
|
280
280
|
return `~${row[0]}`;
|
|
@@ -374,8 +374,12 @@ WHERE oid = $1::regclass`,
|
|
|
374
374
|
micro.logger.info(`${this.slot_name} Replicating ${table.qualifiedName}`);
|
|
375
375
|
const estimatedCount = await this.estimatedCount(db, table);
|
|
376
376
|
let at = 0;
|
|
377
|
+
let lastLogIndex = 0;
|
|
377
378
|
const cursor = await db.stream({ statement: `SELECT * FROM ${table.escapedIdentifier}` });
|
|
378
379
|
let columns: { i: number; name: string }[] = [];
|
|
380
|
+
// pgwire streams rows in chunks.
|
|
381
|
+
// These chunks can be quite small (as little as 16KB), so we don't flush chunks automatically.
|
|
382
|
+
|
|
379
383
|
for await (let chunk of cursor) {
|
|
380
384
|
if (chunk.tag == 'RowDescription') {
|
|
381
385
|
let i = 0;
|
|
@@ -392,22 +396,21 @@ WHERE oid = $1::regclass`,
|
|
|
392
396
|
}
|
|
393
397
|
return q;
|
|
394
398
|
});
|
|
395
|
-
if (
|
|
399
|
+
if (rows.length > 0 && at - lastLogIndex >= 5000) {
|
|
396
400
|
micro.logger.info(`${this.slot_name} Replicating ${table.qualifiedName} ${at}/${estimatedCount}`);
|
|
401
|
+
lastLogIndex = at;
|
|
397
402
|
}
|
|
398
403
|
if (this.abort_signal.aborted) {
|
|
399
404
|
throw new Error(`Aborted initial replication of ${this.slot_name}`);
|
|
400
405
|
}
|
|
401
406
|
|
|
402
407
|
for (let record of WalStream.getQueryData(rows)) {
|
|
408
|
+
// This auto-flushes when the batch reaches its size limit
|
|
403
409
|
await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record });
|
|
404
410
|
}
|
|
405
411
|
at += rows.length;
|
|
406
412
|
Metrics.getInstance().rows_replicated_total.add(rows.length);
|
|
407
413
|
|
|
408
|
-
// pgwire streaming uses reasonable chunk sizes, so we flush at the end
|
|
409
|
-
// of each chunk.
|
|
410
|
-
await batch.flush();
|
|
411
414
|
await touch();
|
|
412
415
|
}
|
|
413
416
|
|
package/src/routes/router.ts
CHANGED
|
@@ -13,7 +13,7 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
|
|
|
13
13
|
authorize: ({ context }) => {
|
|
14
14
|
return {
|
|
15
15
|
authorized: !!context.token_payload,
|
|
16
|
-
errors: ['Authentication required']
|
|
16
|
+
errors: ['Authentication required'].concat(context.token_errors ?? [])
|
|
17
17
|
};
|
|
18
18
|
},
|
|
19
19
|
validator: micro.schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
@@ -350,12 +350,12 @@ export class MongoBucketStorage implements BucketStorageFactory {
|
|
|
350
350
|
|
|
351
351
|
if (!instance) {
|
|
352
352
|
const manager = locks.createMongoLockManager(this.db.locks, {
|
|
353
|
-
|
|
353
|
+
name: `instance-id-insertion-lock`
|
|
354
354
|
});
|
|
355
355
|
|
|
356
356
|
await manager.lock(async () => {
|
|
357
357
|
await this.db.instance.insertOne({
|
|
358
|
-
_id: uuid()
|
|
358
|
+
_id: uuid()
|
|
359
359
|
});
|
|
360
360
|
});
|
|
361
361
|
|
|
@@ -463,10 +463,8 @@ export class MongoBucketBatch implements BucketStorageBatch {
|
|
|
463
463
|
|
|
464
464
|
await this.withTransaction(async () => {
|
|
465
465
|
flushTry += 1;
|
|
466
|
-
if (flushTry ==
|
|
467
|
-
micro.logger.info(`${this.slot_name} ${description}`);
|
|
468
|
-
} else if (flushTry % 10 == 0) {
|
|
469
|
-
micro.logger.info(`${this.slot_name} ${description} ops - try ${flushTry}`);
|
|
466
|
+
if (flushTry % 10 == 0) {
|
|
467
|
+
micro.logger.info(`${this.slot_name} ${description} - try ${flushTry}`);
|
|
470
468
|
}
|
|
471
469
|
if (flushTry > 20 && Date.now() > lastTry) {
|
|
472
470
|
throw new Error('Max transaction tries exceeded');
|
|
@@ -12,7 +12,7 @@ const MAX_BATCH_COUNT = 2000;
|
|
|
12
12
|
/**
|
|
13
13
|
* Maximum size of operations in the batch (estimated).
|
|
14
14
|
*/
|
|
15
|
-
const MAX_RECORD_BATCH_SIZE =
|
|
15
|
+
const MAX_RECORD_BATCH_SIZE = 5_000_000;
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
18
|
* Maximum size of size of current_data documents we lookup at a time.
|
|
@@ -2,6 +2,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
|
|
|
2
2
|
import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
import * as mongo from 'mongodb';
|
|
5
|
+
import * as micro from '@journeyapps-platform/micro';
|
|
5
6
|
|
|
6
7
|
import * as util from '@/util/util-index.js';
|
|
7
8
|
import { SourceTable } from '../SourceTable.js';
|
|
@@ -42,6 +43,11 @@ export class PersistedBatch {
|
|
|
42
43
|
bucketParameters: mongo.AnyBulkWriteOperation<BucketParameterDocument>[] = [];
|
|
43
44
|
currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[] = [];
|
|
44
45
|
|
|
46
|
+
/**
|
|
47
|
+
* For debug logging only.
|
|
48
|
+
*/
|
|
49
|
+
debugLastOpId: bigint | null = null;
|
|
50
|
+
|
|
45
51
|
/**
|
|
46
52
|
* Very rough estimate of transaction size.
|
|
47
53
|
*/
|
|
@@ -75,13 +81,16 @@ export class PersistedBatch {
|
|
|
75
81
|
const checksum = util.hashData(k.table, k.id, recordData);
|
|
76
82
|
this.currentSize += recordData.length + 200;
|
|
77
83
|
|
|
84
|
+
const op_id = options.op_seq.next();
|
|
85
|
+
this.debugLastOpId = op_id;
|
|
86
|
+
|
|
78
87
|
this.bucketData.push({
|
|
79
88
|
insertOne: {
|
|
80
89
|
document: {
|
|
81
90
|
_id: {
|
|
82
91
|
g: this.group_id,
|
|
83
92
|
b: k.bucket,
|
|
84
|
-
o:
|
|
93
|
+
o: op_id
|
|
85
94
|
},
|
|
86
95
|
op: 'PUT',
|
|
87
96
|
source_table: options.table.id,
|
|
@@ -97,13 +106,17 @@ export class PersistedBatch {
|
|
|
97
106
|
|
|
98
107
|
for (let bd of remaining_buckets.values()) {
|
|
99
108
|
// REMOVE
|
|
109
|
+
|
|
110
|
+
const op_id = options.op_seq.next();
|
|
111
|
+
this.debugLastOpId = op_id;
|
|
112
|
+
|
|
100
113
|
this.bucketData.push({
|
|
101
114
|
insertOne: {
|
|
102
115
|
document: {
|
|
103
116
|
_id: {
|
|
104
117
|
g: this.group_id,
|
|
105
118
|
b: bd.bucket,
|
|
106
|
-
o:
|
|
119
|
+
o: op_id
|
|
107
120
|
},
|
|
108
121
|
op: 'REMOVE',
|
|
109
122
|
source_table: options.table.id,
|
|
@@ -145,7 +158,9 @@ export class PersistedBatch {
|
|
|
145
158
|
const binLookup = serializeLookup(result.lookup);
|
|
146
159
|
const hex = binLookup.toString('base64');
|
|
147
160
|
remaining_lookups.delete(hex);
|
|
161
|
+
|
|
148
162
|
const op_id = data.op_seq.next();
|
|
163
|
+
this.debugLastOpId = op_id;
|
|
149
164
|
this.bucketParameters.push({
|
|
150
165
|
insertOne: {
|
|
151
166
|
document: {
|
|
@@ -167,6 +182,7 @@ export class PersistedBatch {
|
|
|
167
182
|
// 2. "REMOVE" entries for any lookup not touched.
|
|
168
183
|
for (let lookup of remaining_lookups.values()) {
|
|
169
184
|
const op_id = data.op_seq.next();
|
|
185
|
+
this.debugLastOpId = op_id;
|
|
170
186
|
this.bucketParameters.push({
|
|
171
187
|
insertOne: {
|
|
172
188
|
document: {
|
|
@@ -237,9 +253,16 @@ export class PersistedBatch {
|
|
|
237
253
|
});
|
|
238
254
|
}
|
|
239
255
|
|
|
256
|
+
micro.logger.info(
|
|
257
|
+
`powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${
|
|
258
|
+
this.currentData.length
|
|
259
|
+
} updates, ${Math.round(this.currentSize / 1024)}kb. Last op_id: ${this.debugLastOpId}`
|
|
260
|
+
);
|
|
261
|
+
|
|
240
262
|
this.bucketData = [];
|
|
241
263
|
this.bucketParameters = [];
|
|
242
264
|
this.currentData = [];
|
|
243
265
|
this.currentSize = 0;
|
|
266
|
+
this.debugLastOpId = null;
|
|
244
267
|
}
|
|
245
268
|
}
|
package/src/storage/mongo/db.ts
CHANGED
|
@@ -45,9 +45,6 @@ export class PowerSyncMongo {
|
|
|
45
45
|
this.client = client;
|
|
46
46
|
|
|
47
47
|
const db = client.db(options?.database, {
|
|
48
|
-
// Note this issue with useBigInt64: https://jira.mongodb.org/browse/NODE-6165
|
|
49
|
-
// Unfortunately there is no workaround if we want to continue using bigint.
|
|
50
|
-
// We selectively disable this in individual queries where we don't need that.
|
|
51
48
|
...BSON_DESERIALIZE_OPTIONS
|
|
52
49
|
});
|
|
53
50
|
this.db = db;
|
|
@@ -121,7 +121,8 @@ export class CompoundConfigCollector {
|
|
|
121
121
|
migrations: baseConfig.migrations,
|
|
122
122
|
telemetry: {
|
|
123
123
|
disable_telemetry_sharing: baseConfig.telemetry?.disable_telemetry_sharing ?? false,
|
|
124
|
-
internal_service_endpoint:
|
|
124
|
+
internal_service_endpoint:
|
|
125
|
+
baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
|
|
125
126
|
},
|
|
126
127
|
slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
|
|
127
128
|
};
|