@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CHANGELOG.md +152 -4
  2. package/dist/api/RouteAPI.d.ts +42 -14
  3. package/dist/api/diagnostics.d.ts +12 -1
  4. package/dist/api/diagnostics.js +36 -22
  5. package/dist/api/diagnostics.js.map +1 -1
  6. package/dist/entry/commands/compact-action.js +30 -22
  7. package/dist/entry/commands/compact-action.js.map +1 -1
  8. package/dist/metrics/metrics-interfaces.d.ts +5 -1
  9. package/dist/metrics/metrics-interfaces.js.map +1 -1
  10. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
  11. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
  12. package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
  13. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
  14. package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
  15. package/dist/metrics/open-telemetry/util.js +7 -3
  16. package/dist/metrics/open-telemetry/util.js.map +1 -1
  17. package/dist/modules/AbstractModule.d.ts +6 -2
  18. package/dist/modules/AbstractModule.js.map +1 -1
  19. package/dist/replication/AbstractReplicator.d.ts +31 -7
  20. package/dist/replication/AbstractReplicator.js +168 -74
  21. package/dist/replication/AbstractReplicator.js.map +1 -1
  22. package/dist/replication/ReplicationModule.d.ts +3 -3
  23. package/dist/replication/ReplicationModule.js.map +1 -1
  24. package/dist/routes/RouterEngine.d.ts +1 -0
  25. package/dist/routes/RouterEngine.js +6 -1
  26. package/dist/routes/RouterEngine.js.map +1 -1
  27. package/dist/routes/configure-fastify.d.ts +104 -56
  28. package/dist/routes/configure-fastify.js +15 -2
  29. package/dist/routes/configure-fastify.js.map +1 -1
  30. package/dist/routes/endpoints/admin.d.ts +42 -24
  31. package/dist/routes/endpoints/admin.js +49 -36
  32. package/dist/routes/endpoints/admin.js.map +1 -1
  33. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  34. package/dist/routes/endpoints/checkpointing.js +65 -7
  35. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  36. package/dist/routes/endpoints/socket-route.js +3 -2
  37. package/dist/routes/endpoints/socket-route.js.map +1 -1
  38. package/dist/routes/endpoints/sync-rules.js +25 -14
  39. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  40. package/dist/routes/endpoints/sync-stream.js +1 -1
  41. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  42. package/dist/runner/teardown.js +8 -8
  43. package/dist/runner/teardown.js.map +1 -1
  44. package/dist/storage/BucketStorageBatch.d.ts +64 -17
  45. package/dist/storage/BucketStorageBatch.js.map +1 -1
  46. package/dist/storage/BucketStorageFactory.d.ts +50 -31
  47. package/dist/storage/BucketStorageFactory.js +10 -20
  48. package/dist/storage/BucketStorageFactory.js.map +1 -1
  49. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  50. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  51. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  52. package/dist/storage/ChecksumCache.d.ts +11 -6
  53. package/dist/storage/ChecksumCache.js +5 -4
  54. package/dist/storage/ChecksumCache.js.map +1 -1
  55. package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
  56. package/dist/storage/ParsedSyncConfigSet.js +2 -0
  57. package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
  58. package/dist/storage/PersistedReplicationStream.d.ts +32 -0
  59. package/dist/storage/PersistedReplicationStream.js +26 -0
  60. package/dist/storage/PersistedReplicationStream.js.map +1 -0
  61. package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
  62. package/dist/storage/PersistedSyncConfigContent.js +138 -0
  63. package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
  64. package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
  65. package/dist/storage/PersistedSyncConfigStatus.js +2 -0
  66. package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
  67. package/dist/storage/ReplicationEventPayload.d.ts +6 -1
  68. package/dist/storage/SourceEntity.d.ts +13 -2
  69. package/dist/storage/SourceTable.d.ts +53 -4
  70. package/dist/storage/SourceTable.js +62 -7
  71. package/dist/storage/SourceTable.js.map +1 -1
  72. package/dist/storage/SourceTableReconciler.d.ts +83 -0
  73. package/dist/storage/SourceTableReconciler.js +95 -0
  74. package/dist/storage/SourceTableReconciler.js.map +1 -0
  75. package/dist/storage/StorageVersionConfig.d.ts +4 -0
  76. package/dist/storage/StorageVersionConfig.js +10 -1
  77. package/dist/storage/StorageVersionConfig.js.map +1 -1
  78. package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
  79. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  80. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  81. package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
  82. package/dist/storage/WriteCheckpointAPI.js +33 -0
  83. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  84. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
  85. package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
  86. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  87. package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
  88. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
  89. package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
  90. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
  91. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
  92. package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
  93. package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
  94. package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
  95. package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
  96. package/dist/storage/storage-index.d.ts +10 -1
  97. package/dist/storage/storage-index.js +10 -1
  98. package/dist/storage/storage-index.js.map +1 -1
  99. package/dist/storage/storage-metrics.js +120 -4
  100. package/dist/storage/storage-metrics.js.map +1 -1
  101. package/dist/sync/BucketChecksumState.d.ts +12 -2
  102. package/dist/sync/BucketChecksumState.js +69 -21
  103. package/dist/sync/BucketChecksumState.js.map +1 -1
  104. package/dist/sync/RequestTracker.d.ts +7 -0
  105. package/dist/sync/RequestTracker.js +12 -0
  106. package/dist/sync/RequestTracker.js.map +1 -1
  107. package/dist/sync/sync.js +184 -43
  108. package/dist/sync/sync.js.map +1 -1
  109. package/dist/sync/util.d.ts +5 -0
  110. package/dist/sync/util.js +19 -1
  111. package/dist/sync/util.js.map +1 -1
  112. package/dist/system/ServiceContext.d.ts +2 -0
  113. package/dist/system/ServiceContext.js +2 -0
  114. package/dist/system/ServiceContext.js.map +1 -1
  115. package/dist/tracing/PerformanceTracer.d.ts +23 -1
  116. package/dist/tracing/PerformanceTracer.js +11 -0
  117. package/dist/tracing/PerformanceTracer.js.map +1 -1
  118. package/dist/util/checkpointing.d.ts +4 -8
  119. package/dist/util/checkpointing.js +1 -16
  120. package/dist/util/checkpointing.js.map +1 -1
  121. package/dist/util/config/compound-config-collector.js +23 -2
  122. package/dist/util/config/compound-config-collector.js.map +1 -1
  123. package/dist/util/config/defaults.d.ts +2 -0
  124. package/dist/util/config/defaults.js +2 -0
  125. package/dist/util/config/defaults.js.map +1 -1
  126. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
  127. package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
  128. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
  129. package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
  130. package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
  131. package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
  132. package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
  133. package/dist/util/config/types.d.ts +4 -1
  134. package/dist/util/lsn.d.ts +4 -0
  135. package/dist/util/lsn.js +10 -0
  136. package/dist/util/lsn.js.map +1 -1
  137. package/dist/util/protocol-types.d.ts +4 -4
  138. package/dist/util/protocol-types.js +3 -2
  139. package/dist/util/protocol-types.js.map +1 -1
  140. package/dist/util/util-index.d.ts +1 -0
  141. package/dist/util/util-index.js +1 -0
  142. package/dist/util/util-index.js.map +1 -1
  143. package/dist/util/utils.d.ts +3 -2
  144. package/dist/util/utils.js +17 -2
  145. package/dist/util/utils.js.map +1 -1
  146. package/dist/util/write-checkpoint-batcher.d.ts +19 -0
  147. package/dist/util/write-checkpoint-batcher.js +100 -0
  148. package/dist/util/write-checkpoint-batcher.js.map +1 -0
  149. package/package.json +7 -7
  150. package/src/api/RouteAPI.ts +43 -14
  151. package/src/api/diagnostics.ts +45 -26
  152. package/src/entry/commands/compact-action.ts +37 -23
  153. package/src/metrics/metrics-interfaces.ts +6 -1
  154. package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
  155. package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
  156. package/src/metrics/open-telemetry/util.ts +7 -3
  157. package/src/modules/AbstractModule.ts +6 -2
  158. package/src/replication/AbstractReplicator.ts +212 -78
  159. package/src/replication/ReplicationModule.ts +2 -3
  160. package/src/routes/RouterEngine.ts +7 -1
  161. package/src/routes/configure-fastify.ts +25 -2
  162. package/src/routes/endpoints/admin.ts +75 -51
  163. package/src/routes/endpoints/checkpointing.ts +79 -8
  164. package/src/routes/endpoints/socket-route.ts +3 -2
  165. package/src/routes/endpoints/sync-rules.ts +26 -14
  166. package/src/routes/endpoints/sync-stream.ts +1 -1
  167. package/src/runner/teardown.ts +8 -8
  168. package/src/storage/BucketStorageBatch.ts +65 -18
  169. package/src/storage/BucketStorageFactory.ts +73 -46
  170. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  171. package/src/storage/ChecksumCache.ts +27 -14
  172. package/src/storage/ParsedSyncConfigSet.ts +25 -0
  173. package/src/storage/PersistedReplicationStream.ts +55 -0
  174. package/src/storage/PersistedSyncConfigContent.ts +207 -0
  175. package/src/storage/PersistedSyncConfigStatus.ts +14 -0
  176. package/src/storage/ReplicationEventPayload.ts +6 -1
  177. package/src/storage/SourceEntity.ts +12 -2
  178. package/src/storage/SourceTable.ts +96 -8
  179. package/src/storage/SourceTableReconciler.ts +188 -0
  180. package/src/storage/StorageVersionConfig.ts +11 -1
  181. package/src/storage/SyncRulesBucketStorage.ts +189 -27
  182. package/src/storage/WriteCheckpointAPI.ts +140 -6
  183. package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
  184. package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
  185. package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
  186. package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
  187. package/src/storage/storage-index.ts +11 -1
  188. package/src/storage/storage-metrics.ts +154 -8
  189. package/src/sync/BucketChecksumState.ts +105 -21
  190. package/src/sync/RequestTracker.ts +20 -0
  191. package/src/sync/sync.ts +214 -50
  192. package/src/sync/util.ts +33 -8
  193. package/src/system/ServiceContext.ts +7 -0
  194. package/src/tracing/PerformanceTracer.ts +41 -1
  195. package/src/util/checkpointing.ts +4 -22
  196. package/src/util/config/compound-config-collector.ts +31 -1
  197. package/src/util/config/defaults.ts +2 -0
  198. package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
  199. package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
  200. package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
  201. package/src/util/config/types.ts +4 -1
  202. package/src/util/lsn.ts +9 -0
  203. package/src/util/protocol-types.ts +3 -2
  204. package/src/util/util-index.ts +1 -0
  205. package/src/util/utils.ts +16 -2
  206. package/src/util/write-checkpoint-batcher.ts +142 -0
  207. package/test/src/AbstractReplicator.test.ts +177 -0
  208. package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
  209. package/test/src/PersistedSyncConfigContent.test.ts +152 -0
  210. package/test/src/checksum_cache.test.ts +22 -0
  211. package/test/src/config.test.ts +142 -0
  212. package/test/src/diagnostics.test.ts +87 -34
  213. package/test/src/module-loader.test.ts +33 -1
  214. package/test/src/routes/admin.test.ts +188 -18
  215. package/test/src/routes/checkpointing.test.ts +54 -0
  216. package/test/src/routes/mocks.ts +6 -2
  217. package/test/src/routes/stream.test.ts +123 -2
  218. package/test/src/source-table-reconciler.test.ts +244 -0
  219. package/test/src/storage/SourceTable.test.ts +92 -0
  220. package/test/src/storage-metrics.test.ts +148 -0
  221. package/test/src/sync/BucketChecksumState.test.ts +98 -12
  222. package/test/src/sync/util.test.ts +28 -1
  223. package/test/src/util/checkpointing.test.ts +249 -0
  224. package/test/src/util/utils.test.ts +23 -0
  225. package/tsconfig.json +1 -4
  226. package/tsconfig.tsbuildinfo +1 -1
  227. package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
  228. package/dist/storage/PersistedSyncRulesContent.js +0 -100
  229. package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
  230. package/src/storage/PersistedSyncRulesContent.ts +0 -174
package/CHANGELOG.md CHANGED
@@ -1,15 +1,163 @@
1
1
  # @powersync/service-core
2
2
 
3
- ## 0.0.0-dev-20260602073133
3
+ ## 0.0.0-dev-20260909133214
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies [70b00a4]
8
+ - @powersync/service-sync-rules@0.0.0-dev-20260909133214
9
+ - @powersync/lib-services-framework@0.0.0-dev-20260909133214
10
+ - @powersync/service-rsocket-router@0.0.0-dev-20260909133214
11
+
12
+ ## 1.26.0
13
+
14
+ ### Minor Changes
15
+
16
+ - 189bd9e: Add storage_version: 4 as a stable storage version.
17
+ - bb08068: Compile event payload queries into serialized sync plans and evaluate them through the shared row-projection implementation. Existing event deployments use one basic, direct-projection query per physical table; filters and other advanced SQL forms are not currently documented or recommended.
18
+
19
+ Edition 3 now validates and applies event `WHERE` clauses. Existing persisted plans and editions 1 and 2 retain the legacy behavior of ignoring filters and emit a warning; deploy a new sync config to enable compiled filtering. Remove event `WHERE` clauses before rolling the service back to an older version.
20
+
21
+ Event evaluation now follows sync stream semantics by returning all payloads from matching queries, including overlapping exact and wildcard sources, or no payloads when filters do not match. Payload query order does not affect persisted equality. Compiled `SELECT *` payloads also include underscore-prefixed columns, matching sync streams.
22
+
23
+ - 332d649: [MongoDB Storage] Support incremental parameter compacting jobs.
24
+ - 33a3c23: Add optional version_label for sync configs.
25
+ - fccdff2: Add object storage metrics, and storage metrics split by sync config.
26
+ - 989d9e1: Scope MongoDB v3 custom write checkpoint records and reads to the stream-assigned event definition that created them. Custom checkpoint mode names the event whose id is resolved through the active sync config's persisted mapping, while event handlers copy the assigned id from replication event payloads when writing checkpoints. This keeps active and incrementally processing checkpoint events separate when their definitions change.
27
+ - 646c0de: Track replication events as MongoDB storage v3 source-table memberships so unchanged event definitions retain their assigned ids while new or changed definitions are resnapshotted during incremental reprocessing.
28
+ - 2407f71: Apply timeouts and abortSignal handling to all S3 operations. The broad timeouts can be configured using the new `storage.object_storage.defaults_mode` option, or the `AWS_DEFAULTS_MODE` environment variable. Upgrade S3 SDK to fix further timeout issues.
29
+ - a997c88: Restructure MongoDB V3 bucket compacting.
30
+
31
+ ### Patch Changes
32
+
33
+ - e11c54b: Add S3 timing info for sync checkpoints.
34
+ - 0077a8d: Fix connection stall after data fetch errors.
35
+ - cc121b3: [MongoDB Storage V3] Force create new replication stream when the old one is invalidated, and fix other consistency issues with V3 replication stream updates.
36
+ - 4e4063b: Optimize runtime type checks, closes https://github.com/powersync-ja/powersync-service/issues/771.
37
+ - 10131ca: Log checkpoint_interrupted when a stream errors while fetching data.
38
+ - 4037e8f: Add storage.default_storage_version config option.
39
+ - dd7a22f: Add the sync config version label to attributed storage metrics while excluding it from shared OTLP telemetry.
40
+ - Updated dependencies [4b6a23e]
41
+ - Updated dependencies [189bd9e]
42
+ - Updated dependencies [bb08068]
43
+ - Updated dependencies [33a3c23]
44
+ - Updated dependencies [fccdff2]
45
+ - Updated dependencies [646c0de]
46
+ - Updated dependencies [4e4063b]
47
+ - Updated dependencies [4037e8f]
48
+ - Updated dependencies [3972bac]
49
+ - @powersync/service-sync-rules@0.41.0
50
+ - @powersync/service-types@0.18.0
51
+ - @powersync/lib-services-framework@0.10.1
52
+ - @powersync/service-rsocket-router@0.2.26
53
+
54
+ ## 1.25.0
55
+
56
+ ### Minor Changes
57
+
58
+ - 798d739: Add source-owned `SourceTable` reconciliation.
59
+
60
+ `resolveTables()` now queries all overlapping persisted candidates and passes them to a
61
+ source-provided reconciler that returns compatible and incompatible tables, can return
62
+ modified compatible copies, and supplies values used for potential new records. Storage persists allowlisted
63
+ source metadata differences but never interprets them. MongoDB v1/v3 and PostgreSQL storage were
64
+ refactored to this candidate-first model (PostgreSQL gains a nullable `source_metadata` JSONB column
65
+ via migration).
66
+
67
+ ### Patch Changes
68
+
69
+ - 27b56cb: Optimization for generating subkey UUIDs.
70
+
71
+ ## 1.24.0
72
+
73
+ ### Minor Changes
74
+
75
+ - 087b61e: Configurable heartbeat_interval_seconds for MongoDB, Postgres, SQL Server.
76
+ - 2189250: Add `/sync/checkpoint-request` for client-supplied checkpoint request ids, previously called write checkpoint ids. The route returns the stored `checkpoint_request_id`, storage now treats managed request ids as monotonic per user/client, custom checkpoint request ids continue to use the existing `checkpoint` field for backwards compatibility, and `checkpoint_requested_at` metadata lets compact jobs remove expired request-derived checkpoint records.
77
+
78
+ This release includes storage migrations for the checkpoint request metadata. Self-hosters should run migrations as part of the upgrade.
79
+
80
+ - 922f974: Add experimental support for storing bucket data chunks on S3.
81
+
82
+ ### Patch Changes
83
+
84
+ - c4860c9: Improve Postgres sync throughput by reading bucket data with ordered, per-bucket index range scans that stop once the requested batch is full.
85
+ - 483415d: Clarify whether too-many-buckets log breakdowns are grouped by sync stream or legacy bucket definition, and include up
86
+ to 100 entries.
87
+ - aab068b: [MongoDB Storage V3] Add has_clear_op field and simplify checksum queries.
88
+ - 37591e9: Lock replication slots before clearing data, preventing concurrent clearing.
89
+ - Updated dependencies [2189250]
90
+ - Updated dependencies [8daa300]
91
+ - Updated dependencies [df5663b]
92
+ - Updated dependencies [be42e25]
93
+ - Updated dependencies [cb4c627]
94
+ - @powersync/lib-services-framework@0.10.0
95
+ - @powersync/service-types@0.17.0
96
+ - @powersync/service-sync-rules@0.40.0
97
+ - @powersync/service-rsocket-router@0.2.25
98
+
99
+ ## 1.23.3
100
+
101
+ ### Patch Changes
102
+
103
+ - ea31f64: Potential fix and improved stack trace for 'evicted' errors.
104
+ - Updated dependencies [ea71bf3]
105
+ - Updated dependencies [edc6ed4]
106
+ - @powersync/service-sync-rules@0.39.0
107
+ - @powersync/lib-services-framework@0.9.8
108
+ - @powersync/service-rsocket-router@0.2.24
109
+
110
+ ## 1.23.2
111
+
112
+ ### Patch Changes
113
+
114
+ - 71d4a0a: Add logs for checkpoint_complete to assist with debugging. Remove "sync lock" logs.
115
+ - e4f683d: [MongoDB Storage] Add experimental option to allow reading data from secondaries.
116
+ - 71d4a0a: Fix edge case where the service can return incomplete data. This could happen when using bucket priorities, and a checkpoint is invalidated by the compact process while syncing.
117
+ - c2edf86: Add `api.parameters.bucket_count_cache_ttl_minutes` configuration option.
118
+ - df9ab1e: Batch concurrent write checkpoint requests and increase request limits to improve throughput.
119
+ - Updated dependencies [a6ae678]
120
+ - Updated dependencies [c2edf86]
121
+ - @powersync/service-sync-rules@0.38.1
122
+ - @powersync/service-types@0.16.1
123
+ - @powersync/lib-services-framework@0.9.7
124
+ - @powersync/service-rsocket-router@0.2.23
125
+
126
+ ## 1.23.1
127
+
128
+ ### Patch Changes
129
+
130
+ - 7e65360: Restructure replication locks for updated sync config.
131
+
132
+ ## 1.23.0
133
+
134
+ ### Minor Changes
135
+
136
+ - a91a08f: [Experimental] Enable incremental reprocessing for MongoDB source + MongoDB storage. This includes significant changes to the v3 storage format.
137
+
138
+ ### Patch Changes
139
+
140
+ - Updated dependencies [a91a08f]
141
+ - Updated dependencies [184c39f]
142
+ - Updated dependencies [c3f75df]
143
+ - Updated dependencies [4bd35ea]
144
+ - @powersync/service-types@0.16.0
145
+ - @powersync/service-sync-rules@0.38.0
146
+ - @powersync/lib-services-framework@0.9.6
147
+ - @powersync/service-rsocket-router@0.2.22
148
+
149
+ ## 1.22.0
4
150
 
5
151
  ### Minor Changes
6
152
 
7
153
  - e2bf1ad: [Internal] rework resolveTables to handle multiple SourceTables.
8
154
  - 92cc83b: Add the experimental `unstable_sqlite_expression_engine` sync config option to evaluate Sync Streams with SQLite.
9
155
  - 15e2466: [MongoDB] Support snapshotting concurrently with streaming in storage v3+.
156
+ - ebeaa3b: [Postgres] Skip storing a redundant copy of replicated rows in `current_data` for tables with `REPLICA IDENTITY FULL`.
10
157
 
11
158
  ### Patch Changes
12
159
 
160
+ - 17fd96b: Fix duration logs for "Processing batch"
13
161
  - 6e2a57e: Refactor HydratedSyncConfig to support multiple SyncConfigs.
14
162
  - ec6df9f: Add a custom Fastify error handler so uncaught errors honour the negotiated `Content-Encoding` (gzip/zstd) instead of returning a header that doesn't match the body.
15
163
  - 99d33d5: Normalize socket addresses to bare hostnames before IP-range validation, so direct-IP literals with any port form are recognized as IPs.
@@ -25,9 +173,9 @@
25
173
  - Updated dependencies [92cc83b]
26
174
  - Updated dependencies [0aab0f9]
27
175
  - Updated dependencies [a94b6c3]
28
- - @powersync/service-sync-rules@0.0.0-dev-20260602073133
29
- - @powersync/lib-services-framework@0.0.0-dev-20260602073133
30
- - @powersync/service-rsocket-router@0.0.0-dev-20260602073133
176
+ - @powersync/service-sync-rules@0.37.0
177
+ - @powersync/lib-services-framework@0.9.5
178
+ - @powersync/service-rsocket-router@0.2.21
31
179
 
32
180
  ## 1.21.0
33
181
 
@@ -1,6 +1,6 @@
1
1
  import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
2
2
  import * as types from '@powersync/service-types';
3
- import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
3
+ import { ParseSyncConfigOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
4
4
  export interface PatternResult {
5
5
  schema: string;
6
6
  pattern: string;
@@ -20,7 +20,8 @@ export interface SlotWalBudgetOptions {
20
20
  slotName: string;
21
21
  }
22
22
  /**
23
- * Describes all the methods currently required to service the sync API endpoints.
23
+ * Source-specific capabilities used by admin routes, sync config validation,
24
+ * checkpointing, diagnostics, and the sync API.
24
25
  */
25
26
  export interface RouteAPI {
26
27
  /**
@@ -33,16 +34,17 @@ export interface RouteAPI {
33
34
  */
34
35
  getConnectionStatus(): Promise<types.ConnectionStatusV2>;
35
36
  /**
36
- * Generates replication table information from a given pattern of tables.
37
+ * Expand sync config table patterns into table metadata from the source connection.
38
+ *
39
+ * This is used by validation and diagnostics paths to explain what will be
40
+ * replicated for the current sync config.
37
41
  *
38
42
  * @param tablePatterns A set of table patterns which typically come from
39
43
  * the tables listed in sync config definitions.
40
44
  *
41
45
  * @param sqlSyncRules
42
- * @returns A result of all the tables and columns which should be replicated
43
- * based off the input patterns. Certain tests are executed on the
44
- * tables to ensure syncing should function according to the input
45
- * pattern. Debug errors and warnings are reported per table.
46
+ * @returns Matching tables or collections, their columns or fields where available,
47
+ * and per-table warnings or errors for the requested table patterns.
46
48
  */
47
49
  getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SyncConfig): Promise<PatternResult[]>;
48
50
  /**
@@ -59,13 +61,26 @@ export interface RouteAPI {
59
61
  /**
60
62
  * Get the current LSN or equivalent replication HEAD position identifier.
61
63
  *
62
- * The position is provided to the callback. After the callback returns,
63
- * the replication head or a greater one will be streamed on the replication stream.
64
+ * The position is provided to the callback so the caller can persist its
65
+ * write-checkpoint mapping before the source adapter forces any required
66
+ * source-side marker or keepalive. The callback returns whether storage
67
+ * actually needs a source marker: when `shouldAdvance` is true the adapter
68
+ * must ensure that the replication stream will observe this position or a
69
+ * greater one, even when the source is otherwise idle. When it is false, the
70
+ * adapter may skip the source marker.
71
+ *
72
+ * Reading the head and forcing the marker happen within a single source
73
+ * session/connection where applicable, so the marker is causally ordered
74
+ * after the head that was handed to the callback.
64
75
  */
65
76
  createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
66
77
  /**
67
- * @returns The schema for tables inside the connected database. This is typically
68
- * used to validate sync config.
78
+ * @returns The connected source schema in service-friendly table and column
79
+ * formats. Strict-schema sources should include actual tables,
80
+ * collections, columns, and types. Schemaless sources should return
81
+ * the best metadata available and document validation limitations in
82
+ * their adapter behavior. This is typically used to validate sync
83
+ * config and generate client schemas.
69
84
  */
70
85
  getConnectionSchema(): Promise<types.DatabaseSchema[]>;
71
86
  /**
@@ -78,8 +93,21 @@ export interface RouteAPI {
78
93
  */
79
94
  shutdown(): Promise<void>;
80
95
  /**
81
- * Get the default schema (or database) when only a table name is specified in sync config.
96
+ * Get source-specific sync config parsing defaults, such as the default
97
+ * schema or database to use when sync config references an unqualified table
98
+ * name.
99
+ *
100
+ * This is used wherever sync config is parsed for this source, including
101
+ * validation, diagnostics, and sync routing.
102
+ */
103
+ getParseSyncRulesOptions(): ParseSyncConfigOptions;
104
+ }
105
+ export interface ReplicationHeadResult<T> {
106
+ response: T;
107
+ /**
108
+ * True when storage needs the source to force a later observable position.
109
+ * False when storage can prove no marker is needed.
82
110
  */
83
- getParseSyncRulesOptions(): ParseSyncRulesOptions;
111
+ shouldAdvance: boolean;
84
112
  }
85
- export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;
113
+ export type ReplicationHeadCallback<T> = (head: string) => Promise<ReplicationHeadResult<T>>;
@@ -12,10 +12,21 @@ export interface DiagnosticsOptions {
12
12
  * If false, uses placeholder values for e.g. initial_replication_done.
13
13
  */
14
14
  live_status: boolean;
15
+ /**
16
+ * Whether this is the active sync config or not.
17
+ */
18
+ active: boolean;
15
19
  /**
16
20
  * Check against the source postgres connection.
17
21
  */
18
22
  check_connection: boolean;
19
23
  }
20
24
  export declare const DEFAULT_DATASOURCE_ID = "default";
21
- export declare function getSyncRulesStatus(bucketStorage: storage.BucketStorageFactory, apiHandler: RouteAPI, sync_rules: storage.PersistedSyncRulesContent | null, options: DiagnosticsOptions): Promise<SyncRulesStatus | undefined>;
25
+ export declare function getSyncRulesStatus(apiHandler: RouteAPI, syncConfig: storage.PersistedSyncConfigContent | null, options: DiagnosticsOptions,
26
+ /**
27
+ * Storage instance for the replication stream of this config.
28
+ *
29
+ * Required to populate live status (snapshot/checkpoint info). The content object
30
+ * itself is no longer a replication stream, so the caller must resolve this.
31
+ */
32
+ systemStorage?: storage.SyncRulesBucketStorage): Promise<SyncRulesStatus | undefined>;
@@ -1,9 +1,16 @@
1
- import { logger } from '@powersync/lib-services-framework';
1
+ import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
2
2
  import { DEFAULT_TAG } from '@powersync/service-sync-rules';
3
3
  import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
4
4
  export const DEFAULT_DATASOURCE_ID = 'default';
5
- export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules, options) {
6
- if (sync_rules == null) {
5
+ export async function getSyncRulesStatus(apiHandler, syncConfig, options,
6
+ /**
7
+ * Storage instance for the replication stream of this config.
8
+ *
9
+ * Required to populate live status (snapshot/checkpoint info). The content object
10
+ * itself is no longer a replication stream, so the caller must resolve this.
11
+ */
12
+ systemStorage) {
13
+ if (syncConfig == null) {
7
14
  return undefined;
8
15
  }
9
16
  const include_content = options.include_content ?? false;
@@ -13,12 +20,18 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
13
20
  let parsed;
14
21
  let persisted;
15
22
  try {
16
- persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
17
- parsed = persisted.syncConfigWithErrors;
23
+ persisted = syncConfig.parsed(apiHandler.getParseSyncRulesOptions());
24
+ // A content object represents a single sync config, so its parsed result has exactly one entry.
25
+ const [singleConfig] = persisted.syncConfigs;
26
+ if (singleConfig == null) {
27
+ throw new ServiceAssertionError('Expected one sync config');
28
+ }
29
+ parsed = singleConfig;
18
30
  }
19
31
  catch (e) {
20
32
  return {
21
- content: include_content ? sync_rules.sync_rules_content : undefined,
33
+ version_label: syncConfig.version_label,
34
+ content: include_content ? syncConfig.sync_rules_content : undefined,
22
35
  connections: [],
23
36
  errors: [{ level: 'fatal', message: e.message, ts: now }]
24
37
  };
@@ -28,8 +41,7 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
28
41
  // This method can run under some situations if no connection is configured yet.
29
42
  // It will return a default tag in such a case. This default tag is not module specific.
30
43
  const tag = sourceConfig.tag ?? DEFAULT_TAG;
31
- const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
32
- const status = await systemStorage?.getStatus();
44
+ const status = live_status ? await systemStorage?.getStatus() : undefined;
33
45
  let replication_lag_bytes = undefined;
34
46
  let slot_wal_budget = undefined;
35
47
  let tables_flat = [];
@@ -57,10 +69,10 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
57
69
  // Ignore
58
70
  logger.warn(`Unable to get replication lag`, e);
59
71
  }
60
- if (apiHandler.getSlotWalBudget && sync_rules.slot_name) {
72
+ if (apiHandler.getSlotWalBudget && syncConfig.replicationStreamName) {
61
73
  try {
62
74
  slot_wal_budget = await apiHandler.getSlotWalBudget({
63
- slotName: sync_rules.slot_name
75
+ slotName: syncConfig.replicationStreamName
64
76
  });
65
77
  }
66
78
  catch (e) {
@@ -103,11 +115,12 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
103
115
  });
104
116
  }
105
117
  const errors = tables_flat.flatMap((info) => info.errors);
106
- if (sync_rules.last_fatal_error) {
118
+ const statusSource = await syncConfig.getSyncConfigStatus();
119
+ if (statusSource?.last_fatal_error) {
107
120
  errors.push({
108
121
  level: 'fatal',
109
- message: sync_rules.last_fatal_error,
110
- ts: sync_rules.last_fatal_error_ts?.toISOString()
122
+ message: statusSource.last_fatal_error,
123
+ ts: statusSource.last_fatal_error_ts?.toISOString()
111
124
  });
112
125
  }
113
126
  errors.push(...syncRuleErrors.map((error) => syncConfigYamlErrorToReplicationError(error, now)));
@@ -127,10 +140,10 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
127
140
  });
128
141
  }
129
142
  }
130
- if (live_status && status?.active) {
143
+ if (live_status && options.active) {
131
144
  // Check replication lag for active replication stream.
132
145
  // Right now we exclude mysql, since it we don't have consistent keepalives for it.
133
- if (sync_rules.last_checkpoint_ts == null && sync_rules.last_keepalive_ts == null) {
146
+ if (statusSource?.last_checkpoint_ts == null && statusSource?.last_keepalive_ts == null) {
134
147
  errors.push({
135
148
  level: 'warning',
136
149
  message: 'No checkpoint found, cannot calculate replication lag',
@@ -138,7 +151,7 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
138
151
  });
139
152
  }
140
153
  else {
141
- const lastTime = Math.max(sync_rules.last_checkpoint_ts?.getTime() ?? 0, sync_rules.last_keepalive_ts?.getTime() ?? 0);
154
+ const lastTime = Math.max(statusSource.last_checkpoint_ts?.getTime() ?? 0, statusSource.last_keepalive_ts?.getTime() ?? 0);
142
155
  const lagSeconds = Math.round((Date.now() - lastTime) / 1000);
143
156
  // On idle instances, keepalive messages are only persisted every 60 seconds.
144
157
  // So we use 5 minutes as a threshold for warnings, and 15 minutes for critical.
@@ -161,17 +174,18 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
161
174
  }
162
175
  }
163
176
  return {
164
- content: include_content ? sync_rules.sync_rules_content : undefined,
177
+ version_label: syncConfig.version_label,
178
+ content: include_content ? syncConfig.sync_rules_content : undefined,
165
179
  connections: [
166
180
  {
167
181
  id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
168
182
  tag: tag,
169
- slot_name: sync_rules.slot_name,
170
- initial_replication_done: status?.snapshot_done ?? false,
183
+ slot_name: syncConfig.replicationStreamName,
184
+ initial_replication_done: status?.snapshotDone ?? false,
171
185
  // TODO: Rename?
172
- last_lsn: status?.checkpoint_lsn ?? undefined,
173
- last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
174
- last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
186
+ last_lsn: status?.resumeLsn ?? undefined,
187
+ last_checkpoint_ts: statusSource?.last_checkpoint_ts?.toISOString(),
188
+ last_keepalive_ts: statusSource?.last_keepalive_ts?.toISOString(),
175
189
  replication_lag_bytes: replication_lag_bytes,
176
190
  wal_status: slot_wal_budget?.wal_status,
177
191
  safe_wal_size: slot_wal_budget?.safe_wal_size,
@@ -1 +1 @@
1
- {"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAwC,MAAM,+BAA+B,CAAC;AAIlG,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AAsB1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,aAA2C,EAC3C,UAAoB,EACpB,UAAoD,EACpD,OAA2B;IAE3B,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IACzD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IACjD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,IAAI,MAA4B,CAAC;IACjC,IAAI,SAAqC,CAAC;IAC1C,IAAI,CAAC;QACH,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACrE,MAAM,GAAG,SAAS,CAAC,oBAAoB,CAAC;IAC1C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACpE,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACzD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;IACxD,gFAAgF;IAChF,wFAAwF;IACxF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,WAAW,CAAC;IAC5C,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtF,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,SAAS,EAAE,CAAC;IAChD,IAAI,qBAAqB,GAAuB,SAAS,CAAC;IAC1D,IAAI,eAAe,GAAkC,SAAS,CAAC;IAE/D,IAAI,WAAW,GAAgB,EAAE,CAAC;IAElC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC1F,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC7C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC,MAAM,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,qBAAqB,GAAG,MAAM,UAAU,CAAC,sBAAsB,CAAC;oBAC9D,aAAa,EAAE,aAAa;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,SAAS;gBACT,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,CAAC,gBAAgB,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;gBACxD,IAAI,CAAC;oBACH,eAAe,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC;wBAClD,QAAQ,EAAE,UAAU,CAAC,SAAS;qBAC/B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QAEtD,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAa,EAAE;YAC7D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,WAAW;oBACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;oBAE9D,YAAY,EAAE,KAAK;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAmB;oBAC7B,aAAa,EAAE,GAAG;oBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,YAAY;iBAC3B,CAAC;gBACF,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,QAAQ;oBACtB,iBAAiB,EAAE,cAAc;oBACjC,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,UAAU,CAAC,gBAAgB;YACpC,EAAE,EAAE,UAAU,CAAC,mBAAmB,EAAE,WAAW,EAAE;SAClD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qCAAqC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAEjG,IACE,eAAe;QACf,eAAe,CAAC,UAAU,KAAK,MAAM;QACrC,eAAe,CAAC,aAAa,IAAI,IAAI;QACrC,eAAe,CAAC,sBAAsB,IAAI,IAAI;QAC9C,eAAe,CAAC,sBAAsB,GAAG,CAAC,EAC1C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,aAAa,GAAG,eAAe,CAAC,sBAAsB,CAAC,GAAG,GAAG,CAAC,CAC3F,CAAC;QACF,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EACL,sBAAsB,SAAS,eAAe;oBAC9C,6DAA6D;oBAC7D,qEAAqE;gBACvE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,WAAW,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;QAClC,uDAAuD;QACvD,mFAAmF;QACnF,IAAI,UAAU,CAAC,kBAAkB,IAAI,IAAI,IAAI,UAAU,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAClF,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,uDAAuD;gBAChE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,UAAU,CAAC,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,EAC7C,UAAU,CAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAC7C,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;YAC9D,6EAA6E;YAC7E,gFAAgF;YAChF,mGAAmG;YACnG,6FAA6F;YAC7F,IAAI,UAAU,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,UAAU,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;QACpE,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,qBAAqB;gBAC5C,GAAG,EAAE,GAAG;gBACR,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,wBAAwB,EAAE,MAAM,EAAE,aAAa,IAAI,KAAK;gBACxD,gBAAgB;gBAChB,QAAQ,EAAE,MAAM,EAAE,cAAc,IAAI,SAAS;gBAC7C,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,EAAE,WAAW,EAAE;gBAChE,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC9D,qBAAqB,EAAE,qBAAqB;gBAC5C,UAAU,EAAE,eAAe,EAAE,UAAU;gBACvC,aAAa,EAAE,eAAe,EAAE,aAAa;gBAC7C,sBAAsB,EAAE,eAAe,EAAE,sBAAsB;gBAC/D,MAAM,EAAE,WAAW;aACpB;SACF;QACD,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAA0B;IAC7C,IAAI,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7B,IAAI,MAAM,GAAuB,EAAE,CAAC;IACpC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,EAAE,WAAW,EAAwC,MAAM,+BAA+B,CAAC;AAIlG,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AA2B1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAAoB,EACpB,UAAqD,EACrD,OAA2B;AAC3B;;;;;GAKG;AACH,aAA8C;IAE9C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IACzD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IACjD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,IAAI,MAA4B,CAAC;IACjC,IAAI,SAAsC,CAAC;IAC3C,IAAI,CAAC;QACH,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACrE,gGAAgG;QAChG,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC,WAAW,CAAC;QAC7C,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,IAAI,qBAAqB,CAAC,0BAA0B,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,GAAG,YAAY,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,aAAa,EAAE,UAAU,CAAC,aAAa;YACvC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACpE,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACzD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;IACxD,gFAAgF;IAChF,wFAAwF;IACxF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,WAAW,CAAC;IAC5C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,qBAAqB,GAAuB,SAAS,CAAC;IAC1D,IAAI,eAAe,GAAkC,SAAS,CAAC;IAE/D,IAAI,WAAW,GAAgB,EAAE,CAAC;IAElC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC1F,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC7C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC,MAAM,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,qBAAqB,GAAG,MAAM,UAAU,CAAC,sBAAsB,CAAC;oBAC9D,aAAa,EAAE,aAAa;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,SAAS;gBACT,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,CAAC,gBAAgB,IAAI,UAAU,CAAC,qBAAqB,EAAE,CAAC;gBACpE,IAAI,CAAC;oBACH,eAAe,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC;wBAClD,QAAQ,EAAE,UAAU,CAAC,qBAAqB;qBAC3C,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QAEtD,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAa,EAAE;YAC7D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,WAAW;oBACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;oBAE9D,YAAY,EAAE,KAAK;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAmB;oBAC7B,aAAa,EAAE,GAAG;oBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,YAAY;iBAC3B,CAAC;gBACF,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,QAAQ;oBACtB,iBAAiB,EAAE,cAAc;oBACjC,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,mBAAmB,EAAE,CAAC;IAE5D,IAAI,YAAY,EAAE,gBAAgB,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,CAAC,gBAAgB;YACtC,EAAE,EAAE,YAAY,CAAC,mBAAmB,EAAE,WAAW,EAAE;SACpD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qCAAqC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAEjG,IACE,eAAe;QACf,eAAe,CAAC,UAAU,KAAK,MAAM;QACrC,eAAe,CAAC,aAAa,IAAI,IAAI;QACrC,eAAe,CAAC,sBAAsB,IAAI,IAAI;QAC9C,eAAe,CAAC,sBAAsB,GAAG,CAAC,EAC1C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,aAAa,GAAG,eAAe,CAAC,sBAAsB,CAAC,GAAG,GAAG,CAAC,CAC3F,CAAC;QACF,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EACL,sBAAsB,SAAS,eAAe;oBAC9C,6DAA6D;oBAC7D,qEAAqE;gBACvE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,uDAAuD;QACvD,mFAAmF;QACnF,IAAI,YAAY,EAAE,kBAAkB,IAAI,IAAI,IAAI,YAAY,EAAE,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACxF,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,uDAAuD;gBAChE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,YAAY,CAAC,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,EAC/C,YAAY,CAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAC/C,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;YAC9D,6EAA6E;YAC7E,gFAAgF;YAChF,mGAAmG;YACnG,6FAA6F;YAC7F,IAAI,UAAU,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,UAAU,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,aAAa,EAAE,UAAU,CAAC,aAAa;QACvC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;QACpE,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,qBAAqB;gBAC5C,GAAG,EAAE,GAAG;gBACR,SAAS,EAAE,UAAU,CAAC,qBAAqB;gBAC3C,wBAAwB,EAAE,MAAM,EAAE,YAAY,IAAI,KAAK;gBACvD,gBAAgB;gBAChB,QAAQ,EAAE,MAAM,EAAE,SAAS,IAAI,SAAS;gBACxC,kBAAkB,EAAE,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE;gBACnE,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE;gBACjE,qBAAqB,EAAE,qBAAqB;gBAC5C,UAAU,EAAE,eAAe,EAAE,UAAU;gBACvC,aAAa,EAAE,eAAe,EAAE,aAAa;gBAC7C,sBAAsB,EAAE,eAAe,EAAE,sBAAsB;gBAC/D,MAAM,EAAE,WAAW;aACpB;SACF;QACD,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAA0B;IAC7C,IAAI,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7B,IAAI,MAAM,GAAuB,EAAE,CAAC;IACpC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -15,12 +15,14 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
15
15
  * Limit to 1024MB overall.
16
16
  */
17
17
  const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
18
+ const MINUTE_MS = 60 * 1000;
18
19
  export function registerCompactAction(program) {
19
20
  const compactCommand = program
20
21
  .command(COMMAND_NAME)
21
22
  .option(`-b, --buckets [buckets]`, 'Full bucket names, comma-separated (e.g., "global[],mybucket[\\"user1\\"]")')
22
23
  .option('-p, --parameter-indexes', 'Compacting parameter indexes. Defaults to set unless --buckets is provided.')
23
- .option('--no-parameter-indexes', 'Disabling compacting parameter indexes.');
24
+ .option('--no-parameter-indexes', 'Disabling compacting parameter indexes.')
25
+ .option('--incremental-only', '[EXPERIMENTAL] Perform incremental compacting only. Parameter compaction runs on supported storage versions.');
24
26
  wrapConfigCommand(compactCommand);
25
27
  return compactCommand.description('Compact storage').action(async (options) => {
26
28
  const buckets = options.buckets
@@ -35,6 +37,7 @@ export function registerCompactAction(program) {
35
37
  process.exit(1);
36
38
  }
37
39
  }
40
+ const incremental = options.incrementalOnly ?? false;
38
41
  let compactParameters = options.parameterIndexes;
39
42
  if (buckets == null) {
40
43
  logger.info('Compacting storage for all buckets...');
@@ -66,28 +69,33 @@ export function registerCompactAction(program) {
66
69
  // Start the storage engine in order to create the appropriate BucketStorage
67
70
  await serviceContext.lifeCycleEngine.start();
68
71
  const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
69
- const active = await bucketStorage.getActiveStorage();
70
- if (active == null) {
71
- logger.info('No active instance to compact');
72
- return;
72
+ const deleteCheckpointRequestsBefore = new Date(Date.now() - config.api_parameters.checkpoint_request_retention_minutes * MINUTE_MS);
73
+ const streams = await bucketStorage.getReplicatingReplicationStreams();
74
+ for (let stream of streams) {
75
+ const storage = bucketStorage.getInstance(stream);
76
+ const compactParameterData = (compactParameters ?? buckets == null) && (!incremental || storage.supportsIncrementalParameterCompaction());
77
+ logger.info(`[${stream.replicationStreamName}] Performing compaction...`);
78
+ if (buckets != null) {
79
+ await storage.compact({
80
+ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
81
+ compactBuckets: buckets,
82
+ compactParameterData,
83
+ incrementalOnly: incremental,
84
+ deleteCheckpointRequestsBefore,
85
+ signal: abortController.signal
86
+ });
87
+ }
88
+ else {
89
+ await storage.compact({
90
+ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
91
+ compactParameterData,
92
+ incrementalOnly: incremental,
93
+ deleteCheckpointRequestsBefore,
94
+ signal: abortController.signal
95
+ });
96
+ }
97
+ logger.info(`[${stream.replicationStreamName}] Successfully compacted storage.`);
73
98
  }
74
- if (buckets != null) {
75
- logger.info('Performing compaction...');
76
- await active.compact({
77
- memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
78
- compactBuckets: buckets,
79
- compactParameterData: compactParameters ?? false,
80
- signal: abortController.signal
81
- });
82
- }
83
- else {
84
- await active.compact({
85
- memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
86
- compactParameterData: compactParameters ?? true,
87
- signal: abortController.signal
88
- });
89
- }
90
- logger.info('Successfully compacted storage.');
91
99
  }
92
100
  catch (e) {
93
101
  logger.error(`Failed to compact:`, e);
@@ -1 +1 @@
1
- {"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO;SAC3B,OAAO,CAAC,YAAY,CAAC;SACrB,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,wBAAwB,EAAE,yCAAyC,CAAC,CAAC;IAE/E,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;YAC7B,EAAE,KAAK,CAAC,GAAG,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5B,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,KAAK,CACV,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;oBAC7C,0FAA0F,CAC7F,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,GAAmB,OAAO,CAAC,gBAAgB,CAAC;QAEjE,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,0BAA0B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,IAAI,MAAM,CAAC,uBAAuB,CAAC;YACxD,WAAW,EAAE,MAAM,CAAC,kBAAkB,CAAC,OAAO;YAC9C,aAAa,EAAE,MAAM;SACtB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;QAEjD,cAAc,CAAC,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE;YACjD,IAAI,EAAE,KAAK,IAAI,EAAE;gBACf,eAAe,CAAC,KAAK,EAAE,CAAC;gBACxB,MAAM,UAAU,CAAC,OAAO,CAAC;YAC3B,CAAC;SACF,CAAC,CAAC;QAEH,8DAA8D;QAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,aAAa,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,4EAA4E;YAC5E,MAAM,cAAc,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,aAAa,CAAC,mBAAmB,CAAC;YAEvE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;gBAC7C,OAAO;YACT,CAAC;YACD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBACxC,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,aAAa,EAAE,uBAAuB;oBACtC,cAAc,EAAE,OAAO;oBACvB,oBAAoB,EAAE,iBAAiB,IAAI,KAAK;oBAChD,MAAM,EAAE,eAAe,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,aAAa,EAAE,uBAAuB;oBACtC,oBAAoB,EAAE,iBAAiB,IAAI,IAAI;oBAC/C,MAAM,EAAE,eAAe,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YACtC,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,6FAA6F;YAC7F,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAC/E,MAAM,SAAS,GAAG,EAAE,GAAG,IAAI,CAAC;AAE5B,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO;SAC3B,OAAO,CAAC,YAAY,CAAC;SACrB,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,wBAAwB,EAAE,yCAAyC,CAAC;SAC3E,MAAM,CACL,oBAAoB,EACpB,8GAA8G,CAC/G,CAAC;IAEJ,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;YAC7B,EAAE,KAAK,CAAC,GAAG,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5B,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,KAAK,CACV,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;oBAC7C,0FAA0F,CAC7F,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAY,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;QAE9D,IAAI,iBAAiB,GAAmB,OAAO,CAAC,gBAAgB,CAAC;QACjE,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,0BAA0B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,IAAI,MAAM,CAAC,uBAAuB,CAAC;YACxD,WAAW,EAAE,MAAM,CAAC,kBAAkB,CAAC,OAAO;YAC9C,aAAa,EAAE,MAAM;SACtB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;QAEjD,cAAc,CAAC,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE;YACjD,IAAI,EAAE,KAAK,IAAI,EAAE;gBACf,eAAe,CAAC,KAAK,EAAE,CAAC;gBACxB,MAAM,UAAU,CAAC,OAAO,CAAC;YAC3B,CAAC;SACF,CAAC,CAAC;QAEH,8DAA8D;QAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,aAAa,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,4EAA4E;YAC5E,MAAM,cAAc,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,aAAa,CAAC,mBAAmB,CAAC;YACvE,MAAM,8BAA8B,GAAG,IAAI,IAAI,CAC7C,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC,oCAAoC,GAAG,SAAS,CACpF,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,gCAAgC,EAAE,CAAC;YACvE,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,OAAO,GAAG,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAClD,MAAM,oBAAoB,GACxB,CAAC,iBAAiB,IAAI,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,IAAI,OAAO,CAAC,sCAAsC,EAAE,CAAC,CAAC;gBAC/G,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,qBAAqB,4BAA4B,CAAC,CAAC;gBAC1E,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;oBACpB,MAAM,OAAO,CAAC,OAAO,CAAC;wBACpB,aAAa,EAAE,uBAAuB;wBACtC,cAAc,EAAE,OAAO;wBACvB,oBAAoB;wBACpB,eAAe,EAAE,WAAW;wBAC5B,8BAA8B;wBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,OAAO,CAAC,OAAO,CAAC;wBACpB,aAAa,EAAE,uBAAuB;wBACtC,oBAAoB;wBACpB,eAAe,EAAE,WAAW;wBAC5B,8BAA8B;wBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,qBAAqB,mCAAmC,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YACtC,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,6FAA6F;YAC7F,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -17,7 +17,11 @@ export interface ObservableGauge {
17
17
  * Set a value provider that provides the value for the gauge at the time of observation.
18
18
  * @param valueProvider
19
19
  */
20
- setValueProvider(valueProvider: () => Promise<number | undefined>): void;
20
+ setValueProvider(valueProvider: () => Promise<number | ObservableGaugeObservation[] | undefined>): void;
21
+ }
22
+ export interface ObservableGaugeObservation {
23
+ value: number;
24
+ attributes?: Record<string, string>;
21
25
  }
22
26
  export declare enum Precision {
23
27
  INT = "int",
@@ -1 +1 @@
1
- {"version":3,"file":"metrics-interfaces.js","sourceRoot":"","sources":["../../src/metrics/metrics-interfaces.ts"],"names":[],"mappings":"AAwBA,MAAM,CAAN,IAAY,SAGX;AAHD,WAAY,SAAS;IACnB,wBAAW,CAAA;IACX,8BAAiB,CAAA;AACnB,CAAC,EAHW,SAAS,KAAT,SAAS,QAGpB"}
1
+ {"version":3,"file":"metrics-interfaces.js","sourceRoot":"","sources":["../../src/metrics/metrics-interfaces.ts"],"names":[],"mappings":"AA6BA,MAAM,CAAN,IAAY,SAGX;AAHD,WAAY,SAAS;IACnB,wBAAW,CAAA;IACX,8BAAiB,CAAA;AACnB,CAAC,EAHW,SAAS,KAAT,SAAS,QAGpB"}
@@ -0,0 +1,36 @@
1
+ import { ExportResult } from '@opentelemetry/core';
2
+ import { AggregationSelector, AggregationTemporalitySelector, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
3
+ /**
4
+ * Decorates a push exporter and removes selected attributes from every metric data point before
5
+ * delegating the export. The input metrics are not mutated, so another reader can export the
6
+ * original attributes.
7
+ *
8
+ * This exporter deliberately does not merge data points when filtering makes their attribute sets
9
+ * identical. Both points are forwarded with the same attributes. There is no generally correct
10
+ * merge operation at this layer: gauges, counters, and histograms have different aggregation
11
+ * semantics. Callers must therefore ensure that an excluded attribute does not distinguish data
12
+ * points for the same metric, resource, and instrumentation scope. Otherwise, the resulting
13
+ * duplicate series identity may be rejected or interpreted unpredictably downstream.
14
+ *
15
+ * Lifecycle methods and aggregation selection are delegated unchanged to the wrapped exporter.
16
+ */
17
+ export declare class MetricAttributeFilteringExporter implements PushMetricExporter {
18
+ /** The exporter that receives the filtered copy of the metrics. */
19
+ private readonly delegate;
20
+ /** Attribute names to remove from every exported data point. */
21
+ private readonly excludedAttributes;
22
+ readonly selectAggregation?: AggregationSelector;
23
+ readonly selectAggregationTemporality?: AggregationTemporalitySelector;
24
+ constructor(
25
+ /** The exporter that receives the filtered copy of the metrics. */
26
+ delegate: PushMetricExporter,
27
+ /** Attribute names to remove from every exported data point. */
28
+ excludedAttributes: ReadonlySet<string>);
29
+ export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void;
30
+ forceFlush(): Promise<void>;
31
+ shutdown(): Promise<void>;
32
+ private filterAttributes;
33
+ private filterMetricAttributes;
34
+ private filterDataPoint;
35
+ private filterDataPointAttributes;
36
+ }