@powersync/service-core 0.0.0-dev-20250827091123 → 0.0.0-dev-20250828090417

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +18 -12
  2. package/dist/api/api-metrics.js +5 -0
  3. package/dist/api/api-metrics.js.map +1 -1
  4. package/dist/metrics/open-telemetry/util.d.ts +0 -3
  5. package/dist/metrics/open-telemetry/util.js +18 -13
  6. package/dist/metrics/open-telemetry/util.js.map +1 -1
  7. package/dist/routes/compression.d.ts +19 -0
  8. package/dist/routes/compression.js +70 -0
  9. package/dist/routes/compression.js.map +1 -0
  10. package/dist/routes/configure-fastify.js.map +1 -1
  11. package/dist/routes/endpoints/socket-route.js +23 -18
  12. package/dist/routes/endpoints/socket-route.js.map +1 -1
  13. package/dist/routes/endpoints/sync-stream.js +14 -24
  14. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  15. package/dist/routes/router.d.ts +3 -3
  16. package/dist/storage/BucketStorage.d.ts +1 -1
  17. package/dist/storage/BucketStorage.js.map +1 -1
  18. package/dist/storage/BucketStorageFactory.d.ts +0 -2
  19. package/dist/storage/ChecksumCache.d.ts +4 -19
  20. package/dist/storage/ChecksumCache.js +4 -0
  21. package/dist/storage/ChecksumCache.js.map +1 -1
  22. package/dist/storage/StorageEngine.d.ts +2 -2
  23. package/dist/storage/StorageEngine.js.map +1 -1
  24. package/dist/storage/StorageProvider.d.ts +1 -3
  25. package/dist/storage/SyncRulesBucketStorage.d.ts +9 -0
  26. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  27. package/dist/storage/storage-index.d.ts +0 -1
  28. package/dist/storage/storage-index.js +0 -1
  29. package/dist/storage/storage-index.js.map +1 -1
  30. package/dist/sync/BucketChecksumState.d.ts +7 -3
  31. package/dist/sync/BucketChecksumState.js +5 -4
  32. package/dist/sync/BucketChecksumState.js.map +1 -1
  33. package/dist/sync/RequestTracker.d.ts +7 -1
  34. package/dist/sync/RequestTracker.js +22 -2
  35. package/dist/sync/RequestTracker.js.map +1 -1
  36. package/dist/sync/sync.d.ts +2 -2
  37. package/dist/sync/sync.js.map +1 -1
  38. package/dist/sync/util.js +1 -1
  39. package/dist/sync/util.js.map +1 -1
  40. package/dist/system/ServiceContext.d.ts +0 -3
  41. package/dist/system/ServiceContext.js +1 -10
  42. package/dist/system/ServiceContext.js.map +1 -1
  43. package/dist/util/utils.d.ts +17 -2
  44. package/dist/util/utils.js +33 -9
  45. package/dist/util/utils.js.map +1 -1
  46. package/package.json +13 -13
  47. package/src/api/api-metrics.ts +6 -0
  48. package/src/metrics/open-telemetry/util.ts +22 -21
  49. package/src/routes/compression.ts +75 -0
  50. package/src/routes/configure-fastify.ts +1 -0
  51. package/src/routes/endpoints/socket-route.ts +24 -19
  52. package/src/routes/endpoints/sync-stream.ts +15 -24
  53. package/src/routes/router.ts +3 -3
  54. package/src/storage/BucketStorage.ts +2 -2
  55. package/src/storage/BucketStorageFactory.ts +0 -2
  56. package/src/storage/ChecksumCache.ts +8 -22
  57. package/src/storage/StorageEngine.ts +3 -3
  58. package/src/storage/StorageProvider.ts +1 -3
  59. package/src/storage/SyncRulesBucketStorage.ts +12 -0
  60. package/src/storage/storage-index.ts +0 -1
  61. package/src/sync/BucketChecksumState.ts +12 -6
  62. package/src/sync/RequestTracker.ts +27 -2
  63. package/src/sync/sync.ts +3 -3
  64. package/src/sync/util.ts +1 -1
  65. package/src/system/ServiceContext.ts +1 -13
  66. package/src/util/utils.ts +55 -11
  67. package/test/src/checksum_cache.test.ts +6 -8
  68. package/test/src/routes/mocks.ts +59 -0
  69. package/test/src/routes/stream.test.ts +84 -0
  70. package/test/src/sync/BucketChecksumState.test.ts +48 -26
  71. package/tsconfig.tsbuildinfo +1 -1
  72. package/dist/events/EventsEngine.d.ts +0 -14
  73. package/dist/events/EventsEngine.js +0 -33
  74. package/dist/events/EventsEngine.js.map +0 -1
  75. package/dist/storage/ReportStorage.d.ts +0 -36
  76. package/dist/storage/ReportStorage.js +0 -2
  77. package/dist/storage/ReportStorage.js.map +0 -1
  78. package/src/events/EventsEngine.ts +0 -38
  79. package/src/storage/ReportStorage.ts +0 -39
@@ -59,22 +59,46 @@ export function checksumsDiff(previous, current) {
59
59
  export function addChecksums(a, b) {
60
60
  return (a + b) & 0xffffffff;
61
61
  }
62
+ export function isPartialChecksum(c) {
63
+ return 'partialChecksum' in c;
64
+ }
62
65
  export function addBucketChecksums(a, b) {
63
- if (b == null) {
64
- return a;
66
+ const checksum = addPartialChecksums(a.bucket, a, b);
67
+ if (isPartialChecksum(checksum)) {
68
+ // Should not happen since a != null
69
+ throw new ServiceAssertionError('Expected full checksum');
65
70
  }
66
- else if (b.isFullChecksum) {
71
+ return checksum;
72
+ }
73
+ export function addPartialChecksums(bucket, a, b) {
74
+ if (a != null && b != null) {
75
+ if (!isPartialChecksum(b)) {
76
+ // Replaces preState
77
+ return b;
78
+ }
79
+ // merge
67
80
  return {
68
- bucket: b.bucket,
69
- count: b.partialCount,
70
- checksum: b.partialChecksum
81
+ bucket,
82
+ checksum: addChecksums(a.checksum, b.partialChecksum),
83
+ count: a.count + b.partialCount
71
84
  };
72
85
  }
86
+ else if (a != null) {
87
+ return {
88
+ bucket,
89
+ checksum: a.checksum,
90
+ count: a.count
91
+ };
92
+ }
93
+ else if (b != null) {
94
+ return b;
95
+ }
73
96
  else {
97
+ // No data found (may still have a previously-cached checksum).
74
98
  return {
75
- bucket: a.bucket,
76
- count: a.count + b.partialCount,
77
- checksum: addChecksums(a.checksum, b.partialChecksum)
99
+ bucket,
100
+ partialChecksum: 0,
101
+ partialCount: 0
78
102
  };
79
103
  }
80
104
  }
@@ -1 +1 @@
1
- {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAM7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAW1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAkB,EAClB,GAAkC;IAElC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAK7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AA2B1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,CAAmC;IACnE,OAAO,iBAAiB,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAA0C;IAC9F,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACrD,IAAI,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,oCAAoC;QACpC,MAAM,IAAI,qBAAqB,CAAC,wBAAwB,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,CAAwB,EACxB,CAA0C;IAE1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,oBAAoB;YACpB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,QAAQ;QACR,OAAO;YACL,MAAM;YACN,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;YACrD,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;SAChC,CAAC;IACJ,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO;YACL,MAAM;YACN,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC;IACJ,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACN,+DAA+D;QAC/D,OAAO;YACL,MAAM;YACN,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAkB,EAClB,GAAkC;IAElC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
package/package.json CHANGED
@@ -5,17 +5,17 @@
5
5
  "publishConfig": {
6
6
  "access": "public"
7
7
  },
8
- "version": "0.0.0-dev-20250827091123",
8
+ "version": "0.0.0-dev-20250828090417",
9
9
  "main": "dist/index.js",
10
10
  "license": "FSL-1.1-ALv2",
11
11
  "type": "module",
12
12
  "dependencies": {
13
13
  "@js-sdsl/ordered-set": "^4.4.2",
14
- "@opentelemetry/api": "~1.9.0",
15
- "@opentelemetry/exporter-metrics-otlp-http": "^0.57.2",
16
- "@opentelemetry/exporter-prometheus": "^0.57.2",
17
- "@opentelemetry/resources": "^1.30.1",
18
- "@opentelemetry/sdk-metrics": "1.30.1",
14
+ "@opentelemetry/api": "^1.9.0",
15
+ "@opentelemetry/exporter-metrics-otlp-http": "^0.203.0",
16
+ "@opentelemetry/exporter-prometheus": "^0.203.0",
17
+ "@opentelemetry/resources": "^2.0.1",
18
+ "@opentelemetry/sdk-metrics": "^2.0.1",
19
19
  "async": "^3.2.4",
20
20
  "async-mutex": "^0.5.0",
21
21
  "bson": "^6.10.3",
@@ -33,18 +33,18 @@
33
33
  "uuid": "^11.1.0",
34
34
  "winston": "^3.13.0",
35
35
  "yaml": "^2.3.2",
36
- "@powersync/lib-services-framework": "0.0.0-dev-20250827091123",
37
- "@powersync/service-jsonbig": "0.0.0-dev-20250827091123",
38
- "@powersync/service-rsocket-router": "0.0.0-dev-20250827091123",
39
- "@powersync/service-sync-rules": "0.0.0-dev-20250827091123",
40
- "@powersync/service-types": "0.0.0-dev-20250827091123"
36
+ "@powersync/lib-services-framework": "0.0.0-dev-20250828090417",
37
+ "@powersync/service-jsonbig": "0.0.0-dev-20250828090417",
38
+ "@powersync/service-rsocket-router": "0.0.0-dev-20250828090417",
39
+ "@powersync/service-sync-rules": "0.0.0-dev-20250828090417",
40
+ "@powersync/service-types": "0.0.0-dev-20250828090417"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@types/async": "^3.2.24",
44
44
  "@types/negotiator": "^0.6.4",
45
45
  "@types/lodash": "^4.17.5",
46
- "fastify": "4.23.2",
47
- "fastify-plugin": "^4.5.1"
46
+ "fastify": "^5.4.0",
47
+ "fastify-plugin": "^5.0.1"
48
48
  },
49
49
  "scripts": {
50
50
  "build": "tsc -b",
@@ -12,6 +12,12 @@ export function createCoreAPIMetrics(engine: MetricsEngine): void {
12
12
  unit: 'bytes'
13
13
  });
14
14
 
15
+ engine.createCounter({
16
+ name: APIMetric.DATA_SENT_BYTES,
17
+ description: 'Size of data sent to clients, after compression if applicable',
18
+ unit: 'bytes'
19
+ });
20
+
15
21
  engine.createCounter({
16
22
  name: APIMetric.OPERATIONS_SYNCED,
17
23
  description: 'Number of operations synced'
@@ -1,17 +1,13 @@
1
- import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
2
- import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
3
1
  import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
4
- import { Resource } from '@opentelemetry/resources';
2
+ import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
3
+ import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
4
+ import { logger } from '@powersync/lib-services-framework';
5
5
  import { ServiceContext } from '../../system/ServiceContext.js';
6
- import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js';
7
6
  import { MetricsFactory } from '../metrics-interfaces.js';
8
- import { logger } from '@powersync/lib-services-framework';
7
+ import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js';
9
8
 
10
9
  import pkg from '../../../package.json' with { type: 'json' };
11
-
12
- export interface RuntimeMetadata {
13
- [key: string]: string | number | undefined;
14
- }
10
+ import { resourceFromAttributes } from '@opentelemetry/resources';
15
11
 
16
12
  export function createOpenTelemetryMetricsFactory(context: ServiceContext): MetricsFactory {
17
13
  const { configuration, lifeCycleEngine, storageEngine } = context;
@@ -43,9 +39,9 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr
43
39
  configuredExporters.push(periodicExporter);
44
40
  }
45
41
 
46
- let resolvedMetadata: (metadata: RuntimeMetadata) => void;
47
- const runtimeMetadata: Promise<RuntimeMetadata> = new Promise((resolve) => {
48
- resolvedMetadata = resolve;
42
+ let resolvedInstanceId: (id: string) => void;
43
+ const instanceIdPromise = new Promise<string>((resolve) => {
44
+ resolvedInstanceId = resolve;
49
45
  });
50
46
 
51
47
  lifeCycleEngine.withLifecycle(null, {
@@ -53,21 +49,26 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr
53
49
  const bucketStorage = storageEngine.activeBucketStorage;
54
50
  try {
55
51
  const instanceId = await bucketStorage.getPowerSyncInstanceId();
56
- resolvedMetadata({ ['instance_id']: instanceId });
52
+ resolvedInstanceId(instanceId);
57
53
  } catch (err) {
58
- resolvedMetadata({ ['instance_id']: 'Unknown' });
54
+ resolvedInstanceId('Unknown');
59
55
  }
60
56
  }
61
57
  });
62
58
 
59
+ const resource = resourceFromAttributes({
60
+ ['service']: 'PowerSync',
61
+ ['service.version']: pkg.version,
62
+ ['instance_id']: instanceIdPromise
63
+ });
64
+
65
+ // This triggers OpenTelemetry to resolve the async attributes (instanceIdPromise).
66
+ // This will never reject, and we don't specifically need to wait for it.
67
+ resource.waitForAsyncAttributes?.();
68
+
63
69
  const meterProvider = new MeterProvider({
64
- resource: new Resource(
65
- {
66
- ['service']: 'PowerSync',
67
- ['service.version']: pkg.version
68
- },
69
- runtimeMetadata
70
- ),
70
+ resource,
71
+
71
72
  readers: configuredExporters
72
73
  });
73
74
 
@@ -0,0 +1,75 @@
1
+ import type Negotiator from 'negotiator';
2
+ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
3
+ import * as zlib from 'node:zlib';
4
+ import { RequestTracker } from '../sync/RequestTracker.js';
5
+
6
+ /**
7
+ * Compress a streamed response.
8
+ *
9
+ * `@fastify/compress` can do something similar, but does not appear to work as well on streamed responses.
10
+ * The manual implementation is simple enough, and gives us more control over the low-level details.
11
+ *
12
+ * @param negotiator Negotiator from the request, to negotiate response encoding
13
+ * @param stream plain-text stream
14
+ * @returns
15
+ */
16
+ export function maybeCompressResponseStream(
17
+ negotiator: Negotiator,
18
+ stream: Readable,
19
+ tracker: RequestTracker
20
+ ): { stream: Readable; encodingHeaders: { 'content-encoding'?: string } } {
21
+ const encoding = (negotiator as any).encoding(['identity', 'gzip', 'zstd'], { preferred: 'zstd' });
22
+ const transform = createCompressionTransform(encoding);
23
+ if (transform == null) {
24
+ // No matching compression supported - leave stream as-is
25
+ return {
26
+ stream,
27
+ encodingHeaders: {}
28
+ };
29
+ } else {
30
+ tracker.setCompressed(encoding);
31
+ return {
32
+ stream: transformStream(stream, transform, tracker),
33
+ encodingHeaders: { 'content-encoding': encoding }
34
+ };
35
+ }
36
+ }
37
+
38
+ function createCompressionTransform(encoding: string | undefined): Transform | null {
39
+ if (encoding == 'zstd') {
40
+ // Available since Node v23.8.0, v22.15.0
41
+ // This does the actual compression in a background thread pool.
42
+ return zlib.createZstdCompress({
43
+ // We need to flush the frame after every new input chunk, to avoid delaying data
44
+ // in the output stream.
45
+ flush: zlib.constants.ZSTD_e_flush,
46
+ params: {
47
+ // Default compression level is 3. We reduce this slightly to limit CPU overhead
48
+ [zlib.constants.ZSTD_c_compressionLevel]: 2
49
+ }
50
+ });
51
+ } else if (encoding == 'gzip') {
52
+ return zlib.createGzip({
53
+ // We need to flush the frame after every new input chunk, to avoid delaying data
54
+ // in the output stream.
55
+ flush: zlib.constants.Z_SYNC_FLUSH
56
+ });
57
+ }
58
+ return null;
59
+ }
60
+
61
+ function transformStream(source: Readable, transform: Transform, tracker: RequestTracker) {
62
+ // pipe does not forward error events automatically, resulting in unhandled error
63
+ // events. This forwards it.
64
+ const out = new PassThrough();
65
+ const trackingTransform = new Transform({
66
+ transform(chunk, _encoding, callback) {
67
+ tracker.addCompressedDataSent(chunk.length);
68
+ callback(null, chunk);
69
+ }
70
+ });
71
+ pipeline(source, transform, trackingTransform, out, (err) => {
72
+ if (err) out.destroy(err);
73
+ });
74
+ return out;
75
+ }
@@ -1,4 +1,5 @@
1
1
  import type fastify from 'fastify';
2
+ import * as uuid from 'uuid';
2
3
 
3
4
  import { registerFastifyNotFoundHandler, registerFastifyRoutes } from './route-register.js';
4
5
 
@@ -1,19 +1,19 @@
1
1
  import { ErrorCode, errors, schema } from '@powersync/lib-services-framework';
2
+ import { RequestParameters } from '@powersync/service-sync-rules';
2
3
 
3
4
  import * as sync from '../../sync/sync-index.js';
4
5
  import * as util from '../../util/util-index.js';
5
6
  import { SocketRouteGenerator } from '../router-socket.js';
6
7
  import { SyncRoutes } from './sync-stream.js';
7
8
 
8
- import { APIMetric, event_types } from '@powersync/service-types';
9
+ import { APIMetric } from '@powersync/service-types';
9
10
 
10
11
  export const syncStreamReactive: SocketRouteGenerator = (router) =>
11
12
  router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
12
13
  validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
13
- handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
14
+ handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal, connection }) => {
14
15
  const { service_context, logger } = context;
15
16
  const { routerEngine, metricsEngine, syncContext } = service_context;
16
- const streamStart = Date.now();
17
17
 
18
18
  logger.defaultMeta = {
19
19
  ...logger.defaultMeta,
@@ -21,15 +21,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
21
21
  client_id: params.client_id,
22
22
  user_agent: context.user_agent
23
23
  };
24
-
25
- const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
26
- client_id: params.client_id ?? '',
27
- user_id: context.user_id!,
28
- user_agent: context.user_agent,
29
- // At this point the token_payload is guaranteed to be present
30
- jwt_exp: new Date(context.token_payload!.exp * 1000),
31
- connected_at: new Date(streamStart)
32
- };
24
+ const streamStart = Date.now();
33
25
 
34
26
  // Best effort guess on why the stream was closed.
35
27
  // We use the `??=` operator everywhere, so that we catch the first relevant
@@ -91,13 +83,19 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
91
83
  });
92
84
 
93
85
  metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
94
- service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData);
95
86
  const tracker = new sync.RequestTracker(metricsEngine);
87
+ if (connection.tracker.encoding) {
88
+ // Must be set before we start the stream
89
+ tracker.setCompressed(connection.tracker.encoding);
90
+ }
96
91
  try {
97
92
  for await (const data of sync.streamResponse({
98
93
  syncContext: syncContext,
99
94
  bucketStorage: bucketStorage,
100
- syncRules: syncRules,
95
+ syncRules: {
96
+ syncRules,
97
+ version: bucketStorage.group_id
98
+ },
101
99
  params: {
102
100
  ...params
103
101
  },
@@ -122,7 +120,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
122
120
  const serialized = sync.syncLineToBson(data);
123
121
  responder.onNext({ data: serialized }, false);
124
122
  requestedN--;
125
- tracker.addDataSynced(serialized.length);
123
+ tracker.addPlaintextDataSynced(serialized.length);
126
124
  }
127
125
 
128
126
  if (requestedN <= 0 && !signal.aborted) {
@@ -159,16 +157,23 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
159
157
  responder.onComplete();
160
158
  removeStopHandler();
161
159
  disposer();
160
+ if (connection.tracker.encoding) {
161
+ // Technically, this may not be unique to this specific stream, since there could be multiple
162
+ // rsocket streams on the same websocket connection. We don't have a way to track compressed bytes
163
+ // on individual streams, and we generally expect 1 stream per connection, so this is a reasonable
164
+ // approximation.
165
+ // If there are multiple streams, bytes written would be split arbitrarily across them, but the
166
+ // total should be correct.
167
+ // For non-compressed cases, this is tracked by the stream itself.
168
+ const socketBytes = connection.tracker.getBytesWritten();
169
+ tracker.addCompressedDataSent(socketBytes);
170
+ }
162
171
  logger.info(`Sync stream complete`, {
163
172
  ...tracker.getLogMeta(),
164
173
  stream_ms: Date.now() - streamStart,
165
174
  close_reason: closeReason ?? 'unknown'
166
175
  });
167
176
  metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
168
- service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
169
- ...sdkData,
170
- disconnected_at: new Date()
171
- });
172
177
  }
173
178
  }
174
179
  });
@@ -1,13 +1,15 @@
1
1
  import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
2
- import { Readable } from 'stream';
3
2
  import Negotiator from 'negotiator';
3
+ import { Readable } from 'stream';
4
4
 
5
5
  import * as sync from '../../sync/sync-index.js';
6
6
  import * as util from '../../util/util-index.js';
7
7
 
8
8
  import { authUser } from '../auth.js';
9
9
  import { routeDefinition } from '../router.js';
10
- import { APIMetric, event_types } from '@powersync/service-types';
10
+
11
+ import { APIMetric } from '@powersync/service-types';
12
+ import { maybeCompressResponseStream } from '../compression.js';
11
13
 
12
14
  export enum SyncRoutes {
13
15
  STREAM = '/sync/stream'
@@ -23,15 +25,16 @@ export const syncStreamed = routeDefinition({
23
25
  authorize: authUser,
24
26
  validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
25
27
  handler: async (payload) => {
26
- const { service_context, logger, token_payload } = payload.context;
28
+ const { service_context, logger } = payload.context;
27
29
  const { routerEngine, storageEngine, metricsEngine, syncContext } = service_context;
28
30
  const headers = payload.request.headers;
29
31
  const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
30
32
  const clientId = payload.params.client_id;
31
33
  const streamStart = Date.now();
34
+ const negotiator = new Negotiator(payload.request);
32
35
  // This falls back to JSON unless there's preference for the bson-stream in the Accept header.
33
36
  const useBson = payload.request.headers.accept
34
- ? new Negotiator(payload.request).mediaType(supportedContentTypes) == concatenatedBsonContentType
37
+ ? negotiator.mediaType(supportedContentTypes) == concatenatedBsonContentType
35
38
  : false;
36
39
 
37
40
  logger.defaultMeta = {
@@ -41,14 +44,6 @@ export const syncStreamed = routeDefinition({
41
44
  user_id: payload.context.user_id,
42
45
  bson: useBson
43
46
  };
44
- const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
45
- client_id: clientId ?? '',
46
- user_id: payload.context.user_id!,
47
- user_agent: userAgent as string,
48
- // At this point the token_payload is guaranteed to be present
49
- jwt_exp: new Date(token_payload!.exp * 1000),
50
- connected_at: new Date(streamStart)
51
- };
52
47
 
53
48
  if (routerEngine.closed) {
54
49
  throw new errors.ServiceError({
@@ -74,11 +69,13 @@ export const syncStreamed = routeDefinition({
74
69
  const tracker = new sync.RequestTracker(metricsEngine);
75
70
  try {
76
71
  metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
77
- service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData);
78
72
  const syncLines = sync.streamResponse({
79
73
  syncContext: syncContext,
80
74
  bucketStorage,
81
- syncRules: syncRules,
75
+ syncRules: {
76
+ syncRules,
77
+ version: bucketStorage.group_id
78
+ },
82
79
  params: payload.params,
83
80
  token: payload.context.token_payload!,
84
81
  tracker,
@@ -88,10 +85,11 @@ export const syncStreamed = routeDefinition({
88
85
  });
89
86
 
90
87
  const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);
91
- const stream = Readable.from(sync.transformToBytesTracked(byteContents, tracker), {
88
+ const plainStream = Readable.from(sync.transformToBytesTracked(byteContents, tracker), {
92
89
  objectMode: false,
93
90
  highWaterMark: 16 * 1024
94
91
  });
92
+ const { stream, encodingHeaders } = maybeCompressResponseStream(negotiator, plainStream, tracker);
95
93
 
96
94
  // Best effort guess on why the stream was closed.
97
95
  // We use the `??=` operator everywhere, so that we catch the first relevant
@@ -126,7 +124,8 @@ export const syncStreamed = routeDefinition({
126
124
  return new router.RouterResponse({
127
125
  status: 200,
128
126
  headers: {
129
- 'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType
127
+ 'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType,
128
+ ...encodingHeaders
130
129
  },
131
130
  data: stream,
132
131
  afterSend: async (details) => {
@@ -135,10 +134,6 @@ export const syncStreamed = routeDefinition({
135
134
  }
136
135
  controller.abort();
137
136
  metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
138
- service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
139
- ...sdkData,
140
- disconnected_at: new Date()
141
- });
142
137
  logger.info(`Sync stream complete`, {
143
138
  ...tracker.getLogMeta(),
144
139
  stream_ms: Date.now() - streamStart,
@@ -149,10 +144,6 @@ export const syncStreamed = routeDefinition({
149
144
  } catch (ex) {
150
145
  controller.abort();
151
146
  metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
152
- service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
153
- ...sdkData,
154
- disconnected_at: new Date()
155
- });
156
147
  }
157
148
  }
158
149
  });
@@ -1,4 +1,4 @@
1
- import { Logger, router } from '@powersync/lib-services-framework';
1
+ import { router, ServiceError, Logger } from '@powersync/lib-services-framework';
2
2
  import type { JwtPayload } from '../auth/auth-index.js';
3
3
  import { ServiceContext } from '../system/ServiceContext.js';
4
4
  import { RouterEngine } from './RouterEngine.js';
@@ -31,11 +31,11 @@ export type BasicRouterRequest = {
31
31
  hostname: string;
32
32
  };
33
33
 
34
- export type ContextProviderOptions = {
34
+ export type ConextProviderOptions = {
35
35
  logger: Logger;
36
36
  };
37
37
 
38
- export type ContextProvider = (request: BasicRouterRequest, options: ContextProviderOptions) => Promise<Context>;
38
+ export type ContextProvider = (request: BasicRouterRequest, options: ConextProviderOptions) => Promise<Context>;
39
39
 
40
40
  export type RequestEndpoint<
41
41
  I,
@@ -39,8 +39,8 @@ export enum SyncRuleState {
39
39
  export const DEFAULT_DOCUMENT_BATCH_LIMIT = 1000;
40
40
  export const DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES = 1 * 1024 * 1024;
41
41
 
42
- export function mergeToast(record: ToastableSqliteRow, persisted: ToastableSqliteRow): ToastableSqliteRow {
43
- const newRecord: ToastableSqliteRow = {};
42
+ export function mergeToast<V>(record: ToastableSqliteRow<V>, persisted: ToastableSqliteRow<V>): ToastableSqliteRow<V> {
43
+ const newRecord: ToastableSqliteRow<V> = {};
44
44
  for (let key in record) {
45
45
  if (typeof record[key] == 'undefined') {
46
46
  newRecord[key] = persisted[key];
@@ -3,7 +3,6 @@ import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent }
3
3
  import { ReplicationEventPayload } from './ReplicationEventPayload.js';
4
4
  import { ReplicationLock } from './ReplicationLock.js';
5
5
  import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
6
- import { ReportStorage } from './ReportStorage.js';
7
6
 
8
7
  /**
9
8
  * Represents a configured storage provider.
@@ -165,4 +164,3 @@ export interface TestStorageOptions {
165
164
  doNotClear?: boolean;
166
165
  }
167
166
  export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
168
- export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;
@@ -1,40 +1,21 @@
1
1
  import { OrderedSet } from '@js-sdsl/ordered-set';
2
2
  import { LRUCache } from 'lru-cache/min';
3
3
  import { BucketChecksum } from '../util/protocol-types.js';
4
- import { addBucketChecksums, ChecksumMap, InternalOpId } from '../util/utils.js';
4
+ import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
5
5
 
6
6
  interface ChecksumFetchContext {
7
7
  fetch(bucket: string): Promise<BucketChecksum>;
8
8
  checkpoint: InternalOpId;
9
9
  }
10
10
 
11
- export interface PartialChecksum {
12
- bucket: string;
13
- /**
14
- * 32-bit unsigned hash.
15
- */
16
- partialChecksum: number;
17
-
18
- /**
19
- * Count of operations - informational only.
20
- */
21
- partialCount: number;
22
-
23
- /**
24
- * True if the queried operations contains (starts with) a CLEAR
25
- * operation, indicating that the partial checksum is the full
26
- * checksum, and must not be added to a previously-cached checksum.
27
- */
28
- isFullChecksum: boolean;
29
- }
30
-
31
11
  export interface FetchPartialBucketChecksum {
32
12
  bucket: string;
33
13
  start?: InternalOpId;
34
14
  end: InternalOpId;
35
15
  }
36
16
 
37
- export type PartialChecksumMap = Map<string, PartialChecksum>;
17
+ export type PartialOrFullChecksum = PartialChecksum | BucketChecksum;
18
+ export type PartialChecksumMap = Map<string, PartialOrFullChecksum>;
38
19
 
39
20
  export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<PartialChecksumMap>;
40
21
 
@@ -127,6 +108,11 @@ export class ChecksumCache {
127
108
  });
128
109
  }
129
110
 
111
+ clear() {
112
+ this.cache.clear();
113
+ this.bucketCheckpoints.clear();
114
+ }
115
+
130
116
  async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]> {
131
117
  const checksums = await this.getChecksumMap(checkpoint, buckets);
132
118
  // Return results in the same order as the request
@@ -1,7 +1,7 @@
1
1
  import { BaseObserver, logger, ServiceError } from '@powersync/lib-services-framework';
2
2
  import { ResolvedPowerSyncConfig } from '../util/util-index.js';
3
3
  import { BucketStorageFactory } from './BucketStorageFactory.js';
4
- import { ActiveStorage, StorageProvider } from './StorageProvider.js';
4
+ import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
5
5
 
6
6
  export type StorageEngineOptions = {
7
7
  configuration: ResolvedPowerSyncConfig;
@@ -14,7 +14,7 @@ export interface StorageEngineListener {
14
14
 
15
15
  export class StorageEngine extends BaseObserver<StorageEngineListener> {
16
16
  // TODO: This will need to revisited when we actually support multiple storage providers.
17
- private storageProviders: Map<string, StorageProvider> = new Map();
17
+ private storageProviders: Map<string, BucketStorageProvider> = new Map();
18
18
  private currentActiveStorage: ActiveStorage | null = null;
19
19
 
20
20
  constructor(private options: StorageEngineOptions) {
@@ -37,7 +37,7 @@ export class StorageEngine extends BaseObserver<StorageEngineListener> {
37
37
  * Register a provider which generates a {@link BucketStorageFactory}
38
38
  * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
39
39
  */
40
- registerProvider(provider: StorageProvider) {
40
+ registerProvider(provider: BucketStorageProvider) {
41
41
  this.storageProviders.set(provider.type, provider);
42
42
  }
43
43