@powersync/service-core 0.0.0-dev-20250827091123 → 0.0.0-dev-20250828090417
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -12
- package/dist/api/api-metrics.js +5 -0
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/metrics/open-telemetry/util.d.ts +0 -3
- package/dist/metrics/open-telemetry/util.js +18 -13
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/routes/compression.d.ts +19 -0
- package/dist/routes/compression.js +70 -0
- package/dist/routes/compression.js.map +1 -0
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +23 -18
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +14 -24
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/router.d.ts +3 -3
- package/dist/storage/BucketStorage.d.ts +1 -1
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +0 -2
- package/dist/storage/ChecksumCache.d.ts +4 -19
- package/dist/storage/ChecksumCache.js +4 -0
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/StorageEngine.d.ts +2 -2
- package/dist/storage/StorageEngine.js.map +1 -1
- package/dist/storage/StorageProvider.d.ts +1 -3
- package/dist/storage/SyncRulesBucketStorage.d.ts +9 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/storage-index.d.ts +0 -1
- package/dist/storage/storage-index.js +0 -1
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +7 -3
- package/dist/sync/BucketChecksumState.js +5 -4
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.d.ts +7 -1
- package/dist/sync/RequestTracker.js +22 -2
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -2
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +0 -3
- package/dist/system/ServiceContext.js +1 -10
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/util/utils.d.ts +17 -2
- package/dist/util/utils.js +33 -9
- package/dist/util/utils.js.map +1 -1
- package/package.json +13 -13
- package/src/api/api-metrics.ts +6 -0
- package/src/metrics/open-telemetry/util.ts +22 -21
- package/src/routes/compression.ts +75 -0
- package/src/routes/configure-fastify.ts +1 -0
- package/src/routes/endpoints/socket-route.ts +24 -19
- package/src/routes/endpoints/sync-stream.ts +15 -24
- package/src/routes/router.ts +3 -3
- package/src/storage/BucketStorage.ts +2 -2
- package/src/storage/BucketStorageFactory.ts +0 -2
- package/src/storage/ChecksumCache.ts +8 -22
- package/src/storage/StorageEngine.ts +3 -3
- package/src/storage/StorageProvider.ts +1 -3
- package/src/storage/SyncRulesBucketStorage.ts +12 -0
- package/src/storage/storage-index.ts +0 -1
- package/src/sync/BucketChecksumState.ts +12 -6
- package/src/sync/RequestTracker.ts +27 -2
- package/src/sync/sync.ts +3 -3
- package/src/sync/util.ts +1 -1
- package/src/system/ServiceContext.ts +1 -13
- package/src/util/utils.ts +55 -11
- package/test/src/checksum_cache.test.ts +6 -8
- package/test/src/routes/mocks.ts +59 -0
- package/test/src/routes/stream.test.ts +84 -0
- package/test/src/sync/BucketChecksumState.test.ts +48 -26
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/events/EventsEngine.d.ts +0 -14
- package/dist/events/EventsEngine.js +0 -33
- package/dist/events/EventsEngine.js.map +0 -1
- package/dist/storage/ReportStorage.d.ts +0 -36
- package/dist/storage/ReportStorage.js +0 -2
- package/dist/storage/ReportStorage.js.map +0 -1
- package/src/events/EventsEngine.ts +0 -38
- package/src/storage/ReportStorage.ts +0 -39
package/dist/util/utils.js
CHANGED
|
@@ -59,22 +59,46 @@ export function checksumsDiff(previous, current) {
|
|
|
59
59
|
export function addChecksums(a, b) {
|
|
60
60
|
return (a + b) & 0xffffffff;
|
|
61
61
|
}
|
|
62
|
+
export function isPartialChecksum(c) {
|
|
63
|
+
return 'partialChecksum' in c;
|
|
64
|
+
}
|
|
62
65
|
export function addBucketChecksums(a, b) {
|
|
63
|
-
|
|
64
|
-
|
|
66
|
+
const checksum = addPartialChecksums(a.bucket, a, b);
|
|
67
|
+
if (isPartialChecksum(checksum)) {
|
|
68
|
+
// Should not happen since a != null
|
|
69
|
+
throw new ServiceAssertionError('Expected full checksum');
|
|
65
70
|
}
|
|
66
|
-
|
|
71
|
+
return checksum;
|
|
72
|
+
}
|
|
73
|
+
export function addPartialChecksums(bucket, a, b) {
|
|
74
|
+
if (a != null && b != null) {
|
|
75
|
+
if (!isPartialChecksum(b)) {
|
|
76
|
+
// Replaces preState
|
|
77
|
+
return b;
|
|
78
|
+
}
|
|
79
|
+
// merge
|
|
67
80
|
return {
|
|
68
|
-
bucket
|
|
69
|
-
|
|
70
|
-
|
|
81
|
+
bucket,
|
|
82
|
+
checksum: addChecksums(a.checksum, b.partialChecksum),
|
|
83
|
+
count: a.count + b.partialCount
|
|
71
84
|
};
|
|
72
85
|
}
|
|
86
|
+
else if (a != null) {
|
|
87
|
+
return {
|
|
88
|
+
bucket,
|
|
89
|
+
checksum: a.checksum,
|
|
90
|
+
count: a.count
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
else if (b != null) {
|
|
94
|
+
return b;
|
|
95
|
+
}
|
|
73
96
|
else {
|
|
97
|
+
// No data found (may still have a previously-cached checksum).
|
|
74
98
|
return {
|
|
75
|
-
bucket
|
|
76
|
-
|
|
77
|
-
|
|
99
|
+
bucket,
|
|
100
|
+
partialChecksum: 0,
|
|
101
|
+
partialCount: 0
|
|
78
102
|
};
|
|
79
103
|
}
|
|
80
104
|
}
|
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAK7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AA2B1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,CAAmC;IACnE,OAAO,iBAAiB,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAA0C;IAC9F,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACrD,IAAI,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,oCAAoC;QACpC,MAAM,IAAI,qBAAqB,CAAC,wBAAwB,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,CAAwB,EACxB,CAA0C;IAE1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,oBAAoB;YACpB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,QAAQ;QACR,OAAO;YACL,MAAM;YACN,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;YACrD,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;SAChC,CAAC;IACJ,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO;YACL,MAAM;YACN,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC;IACJ,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACN,+DAA+D;QAC/D,OAAO;YACL,MAAM;YACN,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAkB,EAClB,GAAkC;IAElC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,17 +5,17 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20250828090417",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-ALv2",
|
|
11
11
|
"type": "module",
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@js-sdsl/ordered-set": "^4.4.2",
|
|
14
|
-
"@opentelemetry/api": "
|
|
15
|
-
"@opentelemetry/exporter-metrics-otlp-http": "^0.
|
|
16
|
-
"@opentelemetry/exporter-prometheus": "^0.
|
|
17
|
-
"@opentelemetry/resources": "^
|
|
18
|
-
"@opentelemetry/sdk-metrics": "
|
|
14
|
+
"@opentelemetry/api": "^1.9.0",
|
|
15
|
+
"@opentelemetry/exporter-metrics-otlp-http": "^0.203.0",
|
|
16
|
+
"@opentelemetry/exporter-prometheus": "^0.203.0",
|
|
17
|
+
"@opentelemetry/resources": "^2.0.1",
|
|
18
|
+
"@opentelemetry/sdk-metrics": "^2.0.1",
|
|
19
19
|
"async": "^3.2.4",
|
|
20
20
|
"async-mutex": "^0.5.0",
|
|
21
21
|
"bson": "^6.10.3",
|
|
@@ -33,18 +33,18 @@
|
|
|
33
33
|
"uuid": "^11.1.0",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
37
|
-
"@powersync/service-jsonbig": "0.0.0-dev-
|
|
38
|
-
"@powersync/service-rsocket-router": "0.0.0-dev-
|
|
39
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
40
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
36
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20250828090417",
|
|
37
|
+
"@powersync/service-jsonbig": "0.0.0-dev-20250828090417",
|
|
38
|
+
"@powersync/service-rsocket-router": "0.0.0-dev-20250828090417",
|
|
39
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20250828090417",
|
|
40
|
+
"@powersync/service-types": "0.0.0-dev-20250828090417"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/async": "^3.2.24",
|
|
44
44
|
"@types/negotiator": "^0.6.4",
|
|
45
45
|
"@types/lodash": "^4.17.5",
|
|
46
|
-
"fastify": "4.
|
|
47
|
-
"fastify-plugin": "^
|
|
46
|
+
"fastify": "^5.4.0",
|
|
47
|
+
"fastify-plugin": "^5.0.1"
|
|
48
48
|
},
|
|
49
49
|
"scripts": {
|
|
50
50
|
"build": "tsc -b",
|
package/src/api/api-metrics.ts
CHANGED
|
@@ -12,6 +12,12 @@ export function createCoreAPIMetrics(engine: MetricsEngine): void {
|
|
|
12
12
|
unit: 'bytes'
|
|
13
13
|
});
|
|
14
14
|
|
|
15
|
+
engine.createCounter({
|
|
16
|
+
name: APIMetric.DATA_SENT_BYTES,
|
|
17
|
+
description: 'Size of data sent to clients, after compression if applicable',
|
|
18
|
+
unit: 'bytes'
|
|
19
|
+
});
|
|
20
|
+
|
|
15
21
|
engine.createCounter({
|
|
16
22
|
name: APIMetric.OPERATIONS_SYNCED,
|
|
17
23
|
description: 'Number of operations synced'
|
|
@@ -1,17 +1,13 @@
|
|
|
1
|
-
import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
|
|
2
|
-
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
|
3
1
|
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
4
|
-
import {
|
|
2
|
+
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
|
|
3
|
+
import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
|
|
4
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
5
5
|
import { ServiceContext } from '../../system/ServiceContext.js';
|
|
6
|
-
import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js';
|
|
7
6
|
import { MetricsFactory } from '../metrics-interfaces.js';
|
|
8
|
-
import {
|
|
7
|
+
import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js';
|
|
9
8
|
|
|
10
9
|
import pkg from '../../../package.json' with { type: 'json' };
|
|
11
|
-
|
|
12
|
-
export interface RuntimeMetadata {
|
|
13
|
-
[key: string]: string | number | undefined;
|
|
14
|
-
}
|
|
10
|
+
import { resourceFromAttributes } from '@opentelemetry/resources';
|
|
15
11
|
|
|
16
12
|
export function createOpenTelemetryMetricsFactory(context: ServiceContext): MetricsFactory {
|
|
17
13
|
const { configuration, lifeCycleEngine, storageEngine } = context;
|
|
@@ -43,9 +39,9 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr
|
|
|
43
39
|
configuredExporters.push(periodicExporter);
|
|
44
40
|
}
|
|
45
41
|
|
|
46
|
-
let
|
|
47
|
-
const
|
|
48
|
-
|
|
42
|
+
let resolvedInstanceId: (id: string) => void;
|
|
43
|
+
const instanceIdPromise = new Promise<string>((resolve) => {
|
|
44
|
+
resolvedInstanceId = resolve;
|
|
49
45
|
});
|
|
50
46
|
|
|
51
47
|
lifeCycleEngine.withLifecycle(null, {
|
|
@@ -53,21 +49,26 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr
|
|
|
53
49
|
const bucketStorage = storageEngine.activeBucketStorage;
|
|
54
50
|
try {
|
|
55
51
|
const instanceId = await bucketStorage.getPowerSyncInstanceId();
|
|
56
|
-
|
|
52
|
+
resolvedInstanceId(instanceId);
|
|
57
53
|
} catch (err) {
|
|
58
|
-
|
|
54
|
+
resolvedInstanceId('Unknown');
|
|
59
55
|
}
|
|
60
56
|
}
|
|
61
57
|
});
|
|
62
58
|
|
|
59
|
+
const resource = resourceFromAttributes({
|
|
60
|
+
['service']: 'PowerSync',
|
|
61
|
+
['service.version']: pkg.version,
|
|
62
|
+
['instance_id']: instanceIdPromise
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
// This triggers OpenTelemetry to resolve the async attributes (instanceIdPromise).
|
|
66
|
+
// This will never reject, and we don't specifically need to wait for it.
|
|
67
|
+
resource.waitForAsyncAttributes?.();
|
|
68
|
+
|
|
63
69
|
const meterProvider = new MeterProvider({
|
|
64
|
-
resource
|
|
65
|
-
|
|
66
|
-
['service']: 'PowerSync',
|
|
67
|
-
['service.version']: pkg.version
|
|
68
|
-
},
|
|
69
|
-
runtimeMetadata
|
|
70
|
-
),
|
|
70
|
+
resource,
|
|
71
|
+
|
|
71
72
|
readers: configuredExporters
|
|
72
73
|
});
|
|
73
74
|
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type Negotiator from 'negotiator';
|
|
2
|
+
import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
3
|
+
import * as zlib from 'node:zlib';
|
|
4
|
+
import { RequestTracker } from '../sync/RequestTracker.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Compress a streamed response.
|
|
8
|
+
*
|
|
9
|
+
* `@fastify/compress` can do something similar, but does not appear to work as well on streamed responses.
|
|
10
|
+
* The manual implementation is simple enough, and gives us more control over the low-level details.
|
|
11
|
+
*
|
|
12
|
+
* @param negotiator Negotiator from the request, to negotiate response encoding
|
|
13
|
+
* @param stream plain-text stream
|
|
14
|
+
* @returns
|
|
15
|
+
*/
|
|
16
|
+
export function maybeCompressResponseStream(
|
|
17
|
+
negotiator: Negotiator,
|
|
18
|
+
stream: Readable,
|
|
19
|
+
tracker: RequestTracker
|
|
20
|
+
): { stream: Readable; encodingHeaders: { 'content-encoding'?: string } } {
|
|
21
|
+
const encoding = (negotiator as any).encoding(['identity', 'gzip', 'zstd'], { preferred: 'zstd' });
|
|
22
|
+
const transform = createCompressionTransform(encoding);
|
|
23
|
+
if (transform == null) {
|
|
24
|
+
// No matching compression supported - leave stream as-is
|
|
25
|
+
return {
|
|
26
|
+
stream,
|
|
27
|
+
encodingHeaders: {}
|
|
28
|
+
};
|
|
29
|
+
} else {
|
|
30
|
+
tracker.setCompressed(encoding);
|
|
31
|
+
return {
|
|
32
|
+
stream: transformStream(stream, transform, tracker),
|
|
33
|
+
encodingHeaders: { 'content-encoding': encoding }
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function createCompressionTransform(encoding: string | undefined): Transform | null {
|
|
39
|
+
if (encoding == 'zstd') {
|
|
40
|
+
// Available since Node v23.8.0, v22.15.0
|
|
41
|
+
// This does the actual compression in a background thread pool.
|
|
42
|
+
return zlib.createZstdCompress({
|
|
43
|
+
// We need to flush the frame after every new input chunk, to avoid delaying data
|
|
44
|
+
// in the output stream.
|
|
45
|
+
flush: zlib.constants.ZSTD_e_flush,
|
|
46
|
+
params: {
|
|
47
|
+
// Default compression level is 3. We reduce this slightly to limit CPU overhead
|
|
48
|
+
[zlib.constants.ZSTD_c_compressionLevel]: 2
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
} else if (encoding == 'gzip') {
|
|
52
|
+
return zlib.createGzip({
|
|
53
|
+
// We need to flush the frame after every new input chunk, to avoid delaying data
|
|
54
|
+
// in the output stream.
|
|
55
|
+
flush: zlib.constants.Z_SYNC_FLUSH
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function transformStream(source: Readable, transform: Transform, tracker: RequestTracker) {
|
|
62
|
+
// pipe does not forward error events automatically, resulting in unhandled error
|
|
63
|
+
// events. This forwards it.
|
|
64
|
+
const out = new PassThrough();
|
|
65
|
+
const trackingTransform = new Transform({
|
|
66
|
+
transform(chunk, _encoding, callback) {
|
|
67
|
+
tracker.addCompressedDataSent(chunk.length);
|
|
68
|
+
callback(null, chunk);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
pipeline(source, transform, trackingTransform, out, (err) => {
|
|
72
|
+
if (err) out.destroy(err);
|
|
73
|
+
});
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import { ErrorCode, errors, schema } from '@powersync/lib-services-framework';
|
|
2
|
+
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
2
3
|
|
|
3
4
|
import * as sync from '../../sync/sync-index.js';
|
|
4
5
|
import * as util from '../../util/util-index.js';
|
|
5
6
|
import { SocketRouteGenerator } from '../router-socket.js';
|
|
6
7
|
import { SyncRoutes } from './sync-stream.js';
|
|
7
8
|
|
|
8
|
-
import { APIMetric
|
|
9
|
+
import { APIMetric } from '@powersync/service-types';
|
|
9
10
|
|
|
10
11
|
export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
11
12
|
router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
|
|
12
13
|
validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
13
|
-
handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
|
|
14
|
+
handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal, connection }) => {
|
|
14
15
|
const { service_context, logger } = context;
|
|
15
16
|
const { routerEngine, metricsEngine, syncContext } = service_context;
|
|
16
|
-
const streamStart = Date.now();
|
|
17
17
|
|
|
18
18
|
logger.defaultMeta = {
|
|
19
19
|
...logger.defaultMeta,
|
|
@@ -21,15 +21,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
21
21
|
client_id: params.client_id,
|
|
22
22
|
user_agent: context.user_agent
|
|
23
23
|
};
|
|
24
|
-
|
|
25
|
-
const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
|
|
26
|
-
client_id: params.client_id ?? '',
|
|
27
|
-
user_id: context.user_id!,
|
|
28
|
-
user_agent: context.user_agent,
|
|
29
|
-
// At this point the token_payload is guaranteed to be present
|
|
30
|
-
jwt_exp: new Date(context.token_payload!.exp * 1000),
|
|
31
|
-
connected_at: new Date(streamStart)
|
|
32
|
-
};
|
|
24
|
+
const streamStart = Date.now();
|
|
33
25
|
|
|
34
26
|
// Best effort guess on why the stream was closed.
|
|
35
27
|
// We use the `??=` operator everywhere, so that we catch the first relevant
|
|
@@ -91,13 +83,19 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
91
83
|
});
|
|
92
84
|
|
|
93
85
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
|
|
94
|
-
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData);
|
|
95
86
|
const tracker = new sync.RequestTracker(metricsEngine);
|
|
87
|
+
if (connection.tracker.encoding) {
|
|
88
|
+
// Must be set before we start the stream
|
|
89
|
+
tracker.setCompressed(connection.tracker.encoding);
|
|
90
|
+
}
|
|
96
91
|
try {
|
|
97
92
|
for await (const data of sync.streamResponse({
|
|
98
93
|
syncContext: syncContext,
|
|
99
94
|
bucketStorage: bucketStorage,
|
|
100
|
-
syncRules:
|
|
95
|
+
syncRules: {
|
|
96
|
+
syncRules,
|
|
97
|
+
version: bucketStorage.group_id
|
|
98
|
+
},
|
|
101
99
|
params: {
|
|
102
100
|
...params
|
|
103
101
|
},
|
|
@@ -122,7 +120,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
122
120
|
const serialized = sync.syncLineToBson(data);
|
|
123
121
|
responder.onNext({ data: serialized }, false);
|
|
124
122
|
requestedN--;
|
|
125
|
-
tracker.
|
|
123
|
+
tracker.addPlaintextDataSynced(serialized.length);
|
|
126
124
|
}
|
|
127
125
|
|
|
128
126
|
if (requestedN <= 0 && !signal.aborted) {
|
|
@@ -159,16 +157,23 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
159
157
|
responder.onComplete();
|
|
160
158
|
removeStopHandler();
|
|
161
159
|
disposer();
|
|
160
|
+
if (connection.tracker.encoding) {
|
|
161
|
+
// Technically, this may not be unique to this specific stream, since there could be multiple
|
|
162
|
+
// rsocket streams on the same websocket connection. We don't have a way to track compressed bytes
|
|
163
|
+
// on individual streams, and we generally expect 1 stream per connection, so this is a reasonable
|
|
164
|
+
// approximation.
|
|
165
|
+
// If there are multiple streams, bytes written would be split arbitrarily across them, but the
|
|
166
|
+
// total should be correct.
|
|
167
|
+
// For non-compressed cases, this is tracked by the stream itself.
|
|
168
|
+
const socketBytes = connection.tracker.getBytesWritten();
|
|
169
|
+
tracker.addCompressedDataSent(socketBytes);
|
|
170
|
+
}
|
|
162
171
|
logger.info(`Sync stream complete`, {
|
|
163
172
|
...tracker.getLogMeta(),
|
|
164
173
|
stream_ms: Date.now() - streamStart,
|
|
165
174
|
close_reason: closeReason ?? 'unknown'
|
|
166
175
|
});
|
|
167
176
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
|
|
168
|
-
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
|
|
169
|
-
...sdkData,
|
|
170
|
-
disconnected_at: new Date()
|
|
171
|
-
});
|
|
172
177
|
}
|
|
173
178
|
}
|
|
174
179
|
});
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
|
|
2
|
-
import { Readable } from 'stream';
|
|
3
2
|
import Negotiator from 'negotiator';
|
|
3
|
+
import { Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
import * as sync from '../../sync/sync-index.js';
|
|
6
6
|
import * as util from '../../util/util-index.js';
|
|
7
7
|
|
|
8
8
|
import { authUser } from '../auth.js';
|
|
9
9
|
import { routeDefinition } from '../router.js';
|
|
10
|
-
|
|
10
|
+
|
|
11
|
+
import { APIMetric } from '@powersync/service-types';
|
|
12
|
+
import { maybeCompressResponseStream } from '../compression.js';
|
|
11
13
|
|
|
12
14
|
export enum SyncRoutes {
|
|
13
15
|
STREAM = '/sync/stream'
|
|
@@ -23,15 +25,16 @@ export const syncStreamed = routeDefinition({
|
|
|
23
25
|
authorize: authUser,
|
|
24
26
|
validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
25
27
|
handler: async (payload) => {
|
|
26
|
-
const { service_context, logger
|
|
28
|
+
const { service_context, logger } = payload.context;
|
|
27
29
|
const { routerEngine, storageEngine, metricsEngine, syncContext } = service_context;
|
|
28
30
|
const headers = payload.request.headers;
|
|
29
31
|
const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
|
|
30
32
|
const clientId = payload.params.client_id;
|
|
31
33
|
const streamStart = Date.now();
|
|
34
|
+
const negotiator = new Negotiator(payload.request);
|
|
32
35
|
// This falls back to JSON unless there's preference for the bson-stream in the Accept header.
|
|
33
36
|
const useBson = payload.request.headers.accept
|
|
34
|
-
?
|
|
37
|
+
? negotiator.mediaType(supportedContentTypes) == concatenatedBsonContentType
|
|
35
38
|
: false;
|
|
36
39
|
|
|
37
40
|
logger.defaultMeta = {
|
|
@@ -41,14 +44,6 @@ export const syncStreamed = routeDefinition({
|
|
|
41
44
|
user_id: payload.context.user_id,
|
|
42
45
|
bson: useBson
|
|
43
46
|
};
|
|
44
|
-
const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
|
|
45
|
-
client_id: clientId ?? '',
|
|
46
|
-
user_id: payload.context.user_id!,
|
|
47
|
-
user_agent: userAgent as string,
|
|
48
|
-
// At this point the token_payload is guaranteed to be present
|
|
49
|
-
jwt_exp: new Date(token_payload!.exp * 1000),
|
|
50
|
-
connected_at: new Date(streamStart)
|
|
51
|
-
};
|
|
52
47
|
|
|
53
48
|
if (routerEngine.closed) {
|
|
54
49
|
throw new errors.ServiceError({
|
|
@@ -74,11 +69,13 @@ export const syncStreamed = routeDefinition({
|
|
|
74
69
|
const tracker = new sync.RequestTracker(metricsEngine);
|
|
75
70
|
try {
|
|
76
71
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
|
|
77
|
-
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData);
|
|
78
72
|
const syncLines = sync.streamResponse({
|
|
79
73
|
syncContext: syncContext,
|
|
80
74
|
bucketStorage,
|
|
81
|
-
syncRules:
|
|
75
|
+
syncRules: {
|
|
76
|
+
syncRules,
|
|
77
|
+
version: bucketStorage.group_id
|
|
78
|
+
},
|
|
82
79
|
params: payload.params,
|
|
83
80
|
token: payload.context.token_payload!,
|
|
84
81
|
tracker,
|
|
@@ -88,10 +85,11 @@ export const syncStreamed = routeDefinition({
|
|
|
88
85
|
});
|
|
89
86
|
|
|
90
87
|
const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);
|
|
91
|
-
const
|
|
88
|
+
const plainStream = Readable.from(sync.transformToBytesTracked(byteContents, tracker), {
|
|
92
89
|
objectMode: false,
|
|
93
90
|
highWaterMark: 16 * 1024
|
|
94
91
|
});
|
|
92
|
+
const { stream, encodingHeaders } = maybeCompressResponseStream(negotiator, plainStream, tracker);
|
|
95
93
|
|
|
96
94
|
// Best effort guess on why the stream was closed.
|
|
97
95
|
// We use the `??=` operator everywhere, so that we catch the first relevant
|
|
@@ -126,7 +124,8 @@ export const syncStreamed = routeDefinition({
|
|
|
126
124
|
return new router.RouterResponse({
|
|
127
125
|
status: 200,
|
|
128
126
|
headers: {
|
|
129
|
-
'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType
|
|
127
|
+
'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType,
|
|
128
|
+
...encodingHeaders
|
|
130
129
|
},
|
|
131
130
|
data: stream,
|
|
132
131
|
afterSend: async (details) => {
|
|
@@ -135,10 +134,6 @@ export const syncStreamed = routeDefinition({
|
|
|
135
134
|
}
|
|
136
135
|
controller.abort();
|
|
137
136
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
|
|
138
|
-
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
|
|
139
|
-
...sdkData,
|
|
140
|
-
disconnected_at: new Date()
|
|
141
|
-
});
|
|
142
137
|
logger.info(`Sync stream complete`, {
|
|
143
138
|
...tracker.getLogMeta(),
|
|
144
139
|
stream_ms: Date.now() - streamStart,
|
|
@@ -149,10 +144,6 @@ export const syncStreamed = routeDefinition({
|
|
|
149
144
|
} catch (ex) {
|
|
150
145
|
controller.abort();
|
|
151
146
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
|
|
152
|
-
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
|
|
153
|
-
...sdkData,
|
|
154
|
-
disconnected_at: new Date()
|
|
155
|
-
});
|
|
156
147
|
}
|
|
157
148
|
}
|
|
158
149
|
});
|
package/src/routes/router.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { router, ServiceError, Logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import type { JwtPayload } from '../auth/auth-index.js';
|
|
3
3
|
import { ServiceContext } from '../system/ServiceContext.js';
|
|
4
4
|
import { RouterEngine } from './RouterEngine.js';
|
|
@@ -31,11 +31,11 @@ export type BasicRouterRequest = {
|
|
|
31
31
|
hostname: string;
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
-
export type
|
|
34
|
+
export type ConextProviderOptions = {
|
|
35
35
|
logger: Logger;
|
|
36
36
|
};
|
|
37
37
|
|
|
38
|
-
export type ContextProvider = (request: BasicRouterRequest, options:
|
|
38
|
+
export type ContextProvider = (request: BasicRouterRequest, options: ConextProviderOptions) => Promise<Context>;
|
|
39
39
|
|
|
40
40
|
export type RequestEndpoint<
|
|
41
41
|
I,
|
|
@@ -39,8 +39,8 @@ export enum SyncRuleState {
|
|
|
39
39
|
export const DEFAULT_DOCUMENT_BATCH_LIMIT = 1000;
|
|
40
40
|
export const DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES = 1 * 1024 * 1024;
|
|
41
41
|
|
|
42
|
-
export function mergeToast(record: ToastableSqliteRow
|
|
43
|
-
const newRecord: ToastableSqliteRow = {};
|
|
42
|
+
export function mergeToast<V>(record: ToastableSqliteRow<V>, persisted: ToastableSqliteRow<V>): ToastableSqliteRow<V> {
|
|
43
|
+
const newRecord: ToastableSqliteRow<V> = {};
|
|
44
44
|
for (let key in record) {
|
|
45
45
|
if (typeof record[key] == 'undefined') {
|
|
46
46
|
newRecord[key] = persisted[key];
|
|
@@ -3,7 +3,6 @@ import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent }
|
|
|
3
3
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
4
4
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
5
5
|
import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
|
|
6
|
-
import { ReportStorage } from './ReportStorage.js';
|
|
7
6
|
|
|
8
7
|
/**
|
|
9
8
|
* Represents a configured storage provider.
|
|
@@ -165,4 +164,3 @@ export interface TestStorageOptions {
|
|
|
165
164
|
doNotClear?: boolean;
|
|
166
165
|
}
|
|
167
166
|
export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
|
|
168
|
-
export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;
|
|
@@ -1,40 +1,21 @@
|
|
|
1
1
|
import { OrderedSet } from '@js-sdsl/ordered-set';
|
|
2
2
|
import { LRUCache } from 'lru-cache/min';
|
|
3
3
|
import { BucketChecksum } from '../util/protocol-types.js';
|
|
4
|
-
import { addBucketChecksums, ChecksumMap, InternalOpId } from '../util/utils.js';
|
|
4
|
+
import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
|
|
5
5
|
|
|
6
6
|
interface ChecksumFetchContext {
|
|
7
7
|
fetch(bucket: string): Promise<BucketChecksum>;
|
|
8
8
|
checkpoint: InternalOpId;
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
-
export interface PartialChecksum {
|
|
12
|
-
bucket: string;
|
|
13
|
-
/**
|
|
14
|
-
* 32-bit unsigned hash.
|
|
15
|
-
*/
|
|
16
|
-
partialChecksum: number;
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* Count of operations - informational only.
|
|
20
|
-
*/
|
|
21
|
-
partialCount: number;
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* True if the queried operations contains (starts with) a CLEAR
|
|
25
|
-
* operation, indicating that the partial checksum is the full
|
|
26
|
-
* checksum, and must not be added to a previously-cached checksum.
|
|
27
|
-
*/
|
|
28
|
-
isFullChecksum: boolean;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
11
|
export interface FetchPartialBucketChecksum {
|
|
32
12
|
bucket: string;
|
|
33
13
|
start?: InternalOpId;
|
|
34
14
|
end: InternalOpId;
|
|
35
15
|
}
|
|
36
16
|
|
|
37
|
-
export type
|
|
17
|
+
export type PartialOrFullChecksum = PartialChecksum | BucketChecksum;
|
|
18
|
+
export type PartialChecksumMap = Map<string, PartialOrFullChecksum>;
|
|
38
19
|
|
|
39
20
|
export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<PartialChecksumMap>;
|
|
40
21
|
|
|
@@ -127,6 +108,11 @@ export class ChecksumCache {
|
|
|
127
108
|
});
|
|
128
109
|
}
|
|
129
110
|
|
|
111
|
+
clear() {
|
|
112
|
+
this.cache.clear();
|
|
113
|
+
this.bucketCheckpoints.clear();
|
|
114
|
+
}
|
|
115
|
+
|
|
130
116
|
async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]> {
|
|
131
117
|
const checksums = await this.getChecksumMap(checkpoint, buckets);
|
|
132
118
|
// Return results in the same order as the request
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { BaseObserver, logger, ServiceError } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ResolvedPowerSyncConfig } from '../util/util-index.js';
|
|
3
3
|
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
4
|
-
import { ActiveStorage,
|
|
4
|
+
import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
|
|
5
5
|
|
|
6
6
|
export type StorageEngineOptions = {
|
|
7
7
|
configuration: ResolvedPowerSyncConfig;
|
|
@@ -14,7 +14,7 @@ export interface StorageEngineListener {
|
|
|
14
14
|
|
|
15
15
|
export class StorageEngine extends BaseObserver<StorageEngineListener> {
|
|
16
16
|
// TODO: This will need to revisited when we actually support multiple storage providers.
|
|
17
|
-
private storageProviders: Map<string,
|
|
17
|
+
private storageProviders: Map<string, BucketStorageProvider> = new Map();
|
|
18
18
|
private currentActiveStorage: ActiveStorage | null = null;
|
|
19
19
|
|
|
20
20
|
constructor(private options: StorageEngineOptions) {
|
|
@@ -37,7 +37,7 @@ export class StorageEngine extends BaseObserver<StorageEngineListener> {
|
|
|
37
37
|
* Register a provider which generates a {@link BucketStorageFactory}
|
|
38
38
|
* given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
|
|
39
39
|
*/
|
|
40
|
-
registerProvider(provider:
|
|
40
|
+
registerProvider(provider: BucketStorageProvider) {
|
|
41
41
|
this.storageProviders.set(provider.type, provider);
|
|
42
42
|
}
|
|
43
43
|
|