@winkjs/composer 0.0.2 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (451) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +106 -80
  3. package/package.json +56 -28
  4. package/src/composer.js +75 -0
  5. package/src/core/batch/aoo-encoder.js +148 -0
  6. package/src/core/batch/index.js +12 -0
  7. package/src/core/batch/sliding-window.js +91 -0
  8. package/src/core/codec/index.js +137 -0
  9. package/src/core/emitter-manager/mqtt/constants.js +102 -0
  10. package/src/core/emitter-manager/mqtt/emitter.js +780 -0
  11. package/src/core/emitter-manager/mqtt/index.js +190 -0
  12. package/src/core/emitter-manager/mqtt/mqtt-store.js +538 -0
  13. package/src/core/emitter-manager/terminal/assert-columns.js +98 -0
  14. package/src/core/emitter-manager/terminal/formatters.js +160 -0
  15. package/src/core/emitter-manager/terminal/index.js +341 -0
  16. package/src/core/env-vars.js +245 -0
  17. package/src/core/mqtt-protocol.js +47 -0
  18. package/src/core/partition-manager/index.js +7 -0
  19. package/src/core/partition-manager/init.js +44 -0
  20. package/src/core/partition-manager/introspect.js +15 -0
  21. package/src/core/partition-manager/publish-to.js +6 -0
  22. package/src/core/partition-manager/recompute.js +5 -0
  23. package/src/core/partition-manager/reset.js +5 -0
  24. package/src/core/partition-manager/update.js +264 -0
  25. package/src/core/partition-manager/validate-flow.js +157 -0
  26. package/src/core/semantics/digest.js +128 -0
  27. package/src/core/semantics/index.js +83 -0
  28. package/src/core/semantics/loader.js +571 -0
  29. package/src/core/semantics/schemas/asset-class-schema.js +218 -0
  30. package/src/core/semantics/schemas/column-schema.js +651 -0
  31. package/src/core/semantics/schemas/enum-schema.js +153 -0
  32. package/src/core/semantics/schemas/index.js +51 -0
  33. package/src/core/semantics/warnings.js +141 -0
  34. package/src/core/shutdown-manager/index.js +289 -0
  35. package/src/core/source-manager/csv/index.js +171 -0
  36. package/src/core/source-manager/csv/start.js +456 -0
  37. package/src/core/source-manager/mqtt/client.js +406 -0
  38. package/src/core/source-manager/mqtt/constants.js +147 -0
  39. package/src/core/source-manager/mqtt/dedup.js +244 -0
  40. package/src/core/source-manager/mqtt/index.js +260 -0
  41. package/src/core/source-manager/mqtt/status.js +546 -0
  42. package/src/core/source-manager/test-harness/comparator.js +226 -0
  43. package/src/core/source-manager/test-harness/field-generator.js +85 -0
  44. package/src/core/source-manager/test-harness/fuzz.js +87 -0
  45. package/src/core/source-manager/test-harness/index.js +115 -0
  46. package/src/core/source-manager/test-harness/prng.js +57 -0
  47. package/src/core/source-manager/test-harness/start.js +232 -0
  48. package/src/core/source-manager/test-harness/validate.js +213 -0
  49. package/src/core/storage-manager/questdb/assert-columns.js +120 -0
  50. package/src/core/storage-manager/questdb/ensure-tables.js +187 -0
  51. package/src/core/storage-manager/questdb/index.js +1251 -0
  52. package/src/core/storage-manager/questdb/persist-plan.js +488 -0
  53. package/src/core/storage-manager/questdb/writers.js +142 -0
  54. package/src/core/test-utils/sealed-record.js +80 -0
  55. package/src/core/test-utils/soak-signature.js +149 -0
  56. package/src/core/test-utils/tcp-proxy.js +106 -0
  57. package/src/core/tunable/helpers.js +254 -0
  58. package/src/core/tunable/index.js +87 -0
  59. package/src/core/utils/console-formatter/index.js +207 -0
  60. package/src/core/utils/create-smart-store-as/index.js +98 -0
  61. package/src/core/utils/create-smart-store-as/sanitize-field.js +46 -0
  62. package/src/core/utils/create-smart-store-as/smart-truncate.js +93 -0
  63. package/src/core/utils/flow/index.js +5 -0
  64. package/src/core/utils/flow/make-collision-checker.js +23 -0
  65. package/src/core/utils/flow/to-kebab.js +20 -0
  66. package/src/core/utils/half-life/index.js +111 -0
  67. package/src/core/utils/node/disable.js +11 -0
  68. package/src/core/utils/node/enable.js +11 -0
  69. package/src/core/utils/node/execute-triggers.js +40 -0
  70. package/src/core/utils/node/index.js +10 -0
  71. package/src/core/utils/node/pause.js +11 -0
  72. package/src/core/utils/node/populate-predicate-inputs.js +14 -0
  73. package/src/core/utils/node/publish-nan.js +19 -0
  74. package/src/core/utils/node/reset-predicate-inputs.js +11 -0
  75. package/src/core/utils/node/unpause.js +11 -0
  76. package/src/core/utils/node/validate-spec.js +22 -0
  77. package/src/core/utils/options/resolve-field-keyed.js +115 -0
  78. package/src/core/utils/quantize/index.js +75 -0
  79. package/src/core/utils/stats/formulas.js +67 -0
  80. package/src/core/utils/stats/index.js +3 -0
  81. package/src/core/utils/validate/composer-validators.js +141 -0
  82. package/src/core/utils/validate/helpers.js +206 -0
  83. package/src/core/utils/validate/index.js +12 -0
  84. package/src/core/utils/validate/type-validators.js +538 -0
  85. package/src/core/utils/validate/validate.js +107 -0
  86. package/src/core/utils/validate/validators.js +177 -0
  87. package/src/core/wiring/assert-handle.js +57 -0
  88. package/src/core/wiring/assert-module.js +63 -0
  89. package/src/core/wiring/index.js +5 -0
  90. package/src/core/wiring/node-type-to-module.js +17 -0
  91. package/src/core/wiring/wire-emitters.js +256 -0
  92. package/src/core/wiring/wire-linear-graph.js +25 -0
  93. package/src/core/wiring/wire-node.js +157 -0
  94. package/src/core/wiring/wire-semantics.js +173 -0
  95. package/src/core/wiring/wire-storages.js +344 -0
  96. package/src/flow/check-output-collisions.js +93 -0
  97. package/src/flow/consts.js +9 -0
  98. package/src/flow/deep-clone-spec.js +64 -0
  99. package/src/flow/driver.js +171 -0
  100. package/src/flow/expand-single-input-array.js +76 -0
  101. package/src/flow/expand-template-spec.js +93 -0
  102. package/src/flow/flow-api-schemas.js +300 -0
  103. package/src/flow/flow.js +658 -0
  104. package/src/flow/for-each.js +225 -0
  105. package/src/flow/get-signature-pattern.js +31 -0
  106. package/src/flow/get-target-array.js +46 -0
  107. package/src/flow/inspect.js +67 -0
  108. package/src/flow/load-node-module.js +81 -0
  109. package/src/flow/make-node-method.js +112 -0
  110. package/src/flow/resolve-pick-by-field.js +77 -0
  111. package/src/flow/resolve-tunables.js +76 -0
  112. package/src/flow/run.js +392 -0
  113. package/src/flow/serialize.js +161 -0
  114. package/src/flow/signatures-schema.js +222 -0
  115. package/src/flow/spec-builder.js +65 -0
  116. package/src/flow/validate-triggers.js +250 -0
  117. package/src/flow/validate.js +261 -0
  118. package/src/nodes/accumulate/index.js +21 -0
  119. package/src/nodes/accumulate/init.js +48 -0
  120. package/src/nodes/accumulate/introspect.js +123 -0
  121. package/src/nodes/accumulate/publish-to.js +34 -0
  122. package/src/nodes/accumulate/recompute.js +27 -0
  123. package/src/nodes/accumulate/reset.js +25 -0
  124. package/src/nodes/accumulate/update.js +42 -0
  125. package/src/nodes/appraise/calibrate.js +64 -0
  126. package/src/nodes/appraise/decision.js +66 -0
  127. package/src/nodes/appraise/deviation.js +140 -0
  128. package/src/nodes/appraise/index.js +23 -0
  129. package/src/nodes/appraise/init.js +165 -0
  130. package/src/nodes/appraise/integrate.js +45 -0
  131. package/src/nodes/appraise/introspect.js +352 -0
  132. package/src/nodes/appraise/publish-to.js +93 -0
  133. package/src/nodes/appraise/receptor.js +69 -0
  134. package/src/nodes/appraise/recompute.js +79 -0
  135. package/src/nodes/appraise/reset.js +53 -0
  136. package/src/nodes/appraise/update.js +130 -0
  137. package/src/nodes/archive/csv/index.js +6 -0
  138. package/src/nodes/archive/csv/init.js +36 -0
  139. package/src/nodes/archive/csv/load.js +70 -0
  140. package/src/nodes/archive/csv/reset.js +14 -0
  141. package/src/nodes/archive/csv/simple-parser.js +216 -0
  142. package/src/nodes/archive/csv/test.js +173 -0
  143. package/src/nodes/archive/csv/update.js +63 -0
  144. package/src/nodes/butterworth-filter/index.js +18 -0
  145. package/src/nodes/butterworth-filter/init.js +184 -0
  146. package/src/nodes/butterworth-filter/introspect.js +228 -0
  147. package/src/nodes/butterworth-filter/publish-to.js +21 -0
  148. package/src/nodes/butterworth-filter/recompute.js +13 -0
  149. package/src/nodes/butterworth-filter/reset.js +30 -0
  150. package/src/nodes/butterworth-filter/update.js +38 -0
  151. package/src/nodes/categorize/index.js +19 -0
  152. package/src/nodes/categorize/init.js +58 -0
  153. package/src/nodes/categorize/introspect.js +177 -0
  154. package/src/nodes/categorize/publish-to.js +29 -0
  155. package/src/nodes/categorize/recompute.js +14 -0
  156. package/src/nodes/categorize/reset.js +16 -0
  157. package/src/nodes/categorize/update.js +53 -0
  158. package/src/nodes/controller/index.js +9 -0
  159. package/src/nodes/controller/init.js +55 -0
  160. package/src/nodes/controller/introspect.js +91 -0
  161. package/src/nodes/controller/publish-to.js +8 -0
  162. package/src/nodes/controller/recompute.js +9 -0
  163. package/src/nodes/controller/reset.js +17 -0
  164. package/src/nodes/controller/update.js +63 -0
  165. package/src/nodes/diff/index.js +19 -0
  166. package/src/nodes/diff/init.js +37 -0
  167. package/src/nodes/diff/introspect.js +132 -0
  168. package/src/nodes/diff/publish-to.js +21 -0
  169. package/src/nodes/diff/recompute.js +12 -0
  170. package/src/nodes/diff/reset.js +12 -0
  171. package/src/nodes/diff/update.js +37 -0
  172. package/src/nodes/digest-moments/formulas.js +12 -0
  173. package/src/nodes/digest-moments/index.js +14 -0
  174. package/src/nodes/digest-moments/init.js +67 -0
  175. package/src/nodes/digest-moments/introspect.js +154 -0
  176. package/src/nodes/digest-moments/publish-to.js +34 -0
  177. package/src/nodes/digest-moments/recompute.js +14 -0
  178. package/src/nodes/digest-moments/reset.js +14 -0
  179. package/src/nodes/digest-moments/update.js +120 -0
  180. package/src/nodes/dwell-time-tracker/index.js +14 -0
  181. package/src/nodes/dwell-time-tracker/init.js +59 -0
  182. package/src/nodes/dwell-time-tracker/introspect.js +137 -0
  183. package/src/nodes/dwell-time-tracker/publish-to.js +22 -0
  184. package/src/nodes/dwell-time-tracker/recompute.js +8 -0
  185. package/src/nodes/dwell-time-tracker/reset.js +23 -0
  186. package/src/nodes/dwell-time-tracker/update.js +103 -0
  187. package/src/nodes/emit-if/helpers.js +94 -0
  188. package/src/nodes/emit-if/index.js +16 -0
  189. package/src/nodes/emit-if/init.js +67 -0
  190. package/src/nodes/emit-if/introspect.js +102 -0
  191. package/src/nodes/emit-if/publish-to.js +13 -0
  192. package/src/nodes/emit-if/recompute.js +12 -0
  193. package/src/nodes/emit-if/reset.js +27 -0
  194. package/src/nodes/emit-if/update.js +93 -0
  195. package/src/nodes/es-correlation/index.js +19 -0
  196. package/src/nodes/es-correlation/init.js +73 -0
  197. package/src/nodes/es-correlation/introspect.js +176 -0
  198. package/src/nodes/es-correlation/publish-to.js +32 -0
  199. package/src/nodes/es-correlation/recompute.js +56 -0
  200. package/src/nodes/es-correlation/reset.js +27 -0
  201. package/src/nodes/es-correlation/update.js +110 -0
  202. package/src/nodes/es-mean/index.js +20 -0
  203. package/src/nodes/es-mean/init.js +58 -0
  204. package/src/nodes/es-mean/introspect.js +129 -0
  205. package/src/nodes/es-mean/publish-to.js +19 -0
  206. package/src/nodes/es-mean/recompute.js +16 -0
  207. package/src/nodes/es-mean/reset.js +24 -0
  208. package/src/nodes/es-mean/update.js +69 -0
  209. package/src/nodes/es-pairwise-correlation/index.js +19 -0
  210. package/src/nodes/es-pairwise-correlation/init.js +98 -0
  211. package/src/nodes/es-pairwise-correlation/introspect.js +187 -0
  212. package/src/nodes/es-pairwise-correlation/publish-to.js +31 -0
  213. package/src/nodes/es-pairwise-correlation/recompute.js +69 -0
  214. package/src/nodes/es-pairwise-correlation/reset.js +29 -0
  215. package/src/nodes/es-pairwise-correlation/update.js +89 -0
  216. package/src/nodes/es-stats/index.js +21 -0
  217. package/src/nodes/es-stats/init.js +105 -0
  218. package/src/nodes/es-stats/introspect.js +157 -0
  219. package/src/nodes/es-stats/publish-to.js +42 -0
  220. package/src/nodes/es-stats/recompute.js +74 -0
  221. package/src/nodes/es-stats/reset.js +49 -0
  222. package/src/nodes/es-stats/update-envelope.js +44 -0
  223. package/src/nodes/es-stats/update-welford.js +64 -0
  224. package/src/nodes/es-stats/update.js +59 -0
  225. package/src/nodes/index.js +83 -0
  226. package/src/nodes/invert-flag/index.js +20 -0
  227. package/src/nodes/invert-flag/init.js +36 -0
  228. package/src/nodes/invert-flag/introspect.js +115 -0
  229. package/src/nodes/invert-flag/publish-to.js +23 -0
  230. package/src/nodes/invert-flag/recompute.js +12 -0
  231. package/src/nodes/invert-flag/reset.js +12 -0
  232. package/src/nodes/invert-flag/update.js +32 -0
  233. package/src/nodes/kalman1d/index.js +20 -0
  234. package/src/nodes/kalman1d/init.js +103 -0
  235. package/src/nodes/kalman1d/introspect.js +250 -0
  236. package/src/nodes/kalman1d/publish-to.js +34 -0
  237. package/src/nodes/kalman1d/recompute.js +23 -0
  238. package/src/nodes/kalman1d/reset.js +29 -0
  239. package/src/nodes/kalman1d/update.js +144 -0
  240. package/src/nodes/kernel/index.js +18 -0
  241. package/src/nodes/kernel/init.js +66 -0
  242. package/src/nodes/kernel/introspect.js +163 -0
  243. package/src/nodes/kernel/presets.js +74 -0
  244. package/src/nodes/kernel/publish-to.js +25 -0
  245. package/src/nodes/kernel/recompute.js +14 -0
  246. package/src/nodes/kernel/reset.js +20 -0
  247. package/src/nodes/kernel/update.js +69 -0
  248. package/src/nodes/lag/index.js +21 -0
  249. package/src/nodes/lag/init.js +87 -0
  250. package/src/nodes/lag/introspect.js +202 -0
  251. package/src/nodes/lag/publish-to.js +64 -0
  252. package/src/nodes/lag/recompute.js +33 -0
  253. package/src/nodes/lag/reset.js +49 -0
  254. package/src/nodes/lag/update.js +133 -0
  255. package/src/nodes/median3/index.js +19 -0
  256. package/src/nodes/median3/init.js +40 -0
  257. package/src/nodes/median3/introspect.js +116 -0
  258. package/src/nodes/median3/publish-to.js +21 -0
  259. package/src/nodes/median3/recompute.js +12 -0
  260. package/src/nodes/median3/reset.js +15 -0
  261. package/src/nodes/median3/update.js +58 -0
  262. package/src/nodes/moments-digest/copy-from.js +12 -0
  263. package/src/nodes/moments-digest/flush.js +17 -0
  264. package/src/nodes/moments-digest/index.js +132 -0
  265. package/src/nodes/moments-digest/init.js +84 -0
  266. package/src/nodes/moments-digest/introspect.js +122 -0
  267. package/src/nodes/moments-digest/prelude.js +28 -0
  268. package/src/nodes/moments-digest/publish-to.js +31 -0
  269. package/src/nodes/moments-digest/recompute.js +9 -0
  270. package/src/nodes/moments-digest/reset.js +20 -0
  271. package/src/nodes/moments-digest/update.js +182 -0
  272. package/src/nodes/node-names.js +67 -0
  273. package/src/nodes/page-hinkley/index.js +20 -0
  274. package/src/nodes/page-hinkley/init.js +66 -0
  275. package/src/nodes/page-hinkley/introspect.js +193 -0
  276. package/src/nodes/page-hinkley/publish-to.js +28 -0
  277. package/src/nodes/page-hinkley/recompute.js +14 -0
  278. package/src/nodes/page-hinkley/reset.js +23 -0
  279. package/src/nodes/page-hinkley/update.js +105 -0
  280. package/src/nodes/pass-if/index.js +21 -0
  281. package/src/nodes/pass-if/init.js +37 -0
  282. package/src/nodes/pass-if/introspect.js +93 -0
  283. package/src/nodes/pass-if/publish-to.js +15 -0
  284. package/src/nodes/pass-if/recompute.js +15 -0
  285. package/src/nodes/pass-if/reset.js +18 -0
  286. package/src/nodes/pass-if/update.js +37 -0
  287. package/src/nodes/persist-if/helpers.js +111 -0
  288. package/src/nodes/persist-if/index.js +9 -0
  289. package/src/nodes/persist-if/init.js +98 -0
  290. package/src/nodes/persist-if/introspect.js +106 -0
  291. package/src/nodes/persist-if/publish-to.js +21 -0
  292. package/src/nodes/persist-if/recompute.js +20 -0
  293. package/src/nodes/persist-if/reset.js +35 -0
  294. package/src/nodes/persist-if/update.js +112 -0
  295. package/src/nodes/persistence-check/index.js +14 -0
  296. package/src/nodes/persistence-check/init.js +38 -0
  297. package/src/nodes/persistence-check/introspect.js +144 -0
  298. package/src/nodes/persistence-check/publish-to.js +11 -0
  299. package/src/nodes/persistence-check/recompute.js +7 -0
  300. package/src/nodes/persistence-check/reset.js +12 -0
  301. package/src/nodes/persistence-check/update.js +57 -0
  302. package/src/nodes/process-index/index.js +15 -0
  303. package/src/nodes/process-index/init.js +48 -0
  304. package/src/nodes/process-index/introspect.js +182 -0
  305. package/src/nodes/process-index/publish-to.js +29 -0
  306. package/src/nodes/process-index/recompute.js +8 -0
  307. package/src/nodes/process-index/reset.js +8 -0
  308. package/src/nodes/process-index/update.js +75 -0
  309. package/src/nodes/ratio/index.js +14 -0
  310. package/src/nodes/ratio/init.js +37 -0
  311. package/src/nodes/ratio/introspect.js +155 -0
  312. package/src/nodes/ratio/publish-to.js +15 -0
  313. package/src/nodes/ratio/recompute.js +7 -0
  314. package/src/nodes/ratio/reset.js +7 -0
  315. package/src/nodes/ratio/update.js +47 -0
  316. package/src/nodes/sanitize/index.js +82 -0
  317. package/src/nodes/sanitize/init.js +82 -0
  318. package/src/nodes/sanitize/introspect.js +194 -0
  319. package/src/nodes/sanitize/publish-to.js +15 -0
  320. package/src/nodes/sanitize/recompute.js +8 -0
  321. package/src/nodes/sanitize/reset.js +14 -0
  322. package/src/nodes/sanitize/update.js +102 -0
  323. package/src/nodes/spike-guard/index.js +14 -0
  324. package/src/nodes/spike-guard/init.js +47 -0
  325. package/src/nodes/spike-guard/introspect.js +130 -0
  326. package/src/nodes/spike-guard/publish-to.js +35 -0
  327. package/src/nodes/spike-guard/recompute.js +14 -0
  328. package/src/nodes/spike-guard/reset.js +19 -0
  329. package/src/nodes/spike-guard/update.js +89 -0
  330. package/src/nodes/sse-emitter/index.js +7 -0
  331. package/src/nodes/sse-emitter/init.js +105 -0
  332. package/src/nodes/sse-emitter/introspect.js +134 -0
  333. package/src/nodes/sse-emitter/publish-to.js +6 -0
  334. package/src/nodes/sse-emitter/recompute.js +16 -0
  335. package/src/nodes/sse-emitter/reset.js +19 -0
  336. package/src/nodes/sse-emitter/update.js +69 -0
  337. package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +36 -0
  338. package/src/nodes/sse-emitter/utils/build-message.js +31 -0
  339. package/src/nodes/sse-emitter/utils/connection-manager.js +45 -0
  340. package/src/nodes/sse-emitter/utils/endpoint-initializer.js +20 -0
  341. package/src/nodes/sse-emitter/utils/health-monitor.js +38 -0
  342. package/src/nodes/sse-emitter/utils/qos-handler.js +41 -0
  343. package/src/nodes/state-change-detector/helpers.js +35 -0
  344. package/src/nodes/state-change-detector/index.js +12 -0
  345. package/src/nodes/state-change-detector/init.js +56 -0
  346. package/src/nodes/state-change-detector/introspect.js +146 -0
  347. package/src/nodes/state-change-detector/publish-to.js +20 -0
  348. package/src/nodes/state-change-detector/recompute.js +6 -0
  349. package/src/nodes/state-change-detector/reset.js +20 -0
  350. package/src/nodes/state-change-detector/update.js +97 -0
  351. package/src/nodes/sw-stats/index.js +14 -0
  352. package/src/nodes/sw-stats/init.js +51 -0
  353. package/src/nodes/sw-stats/introspect.js +131 -0
  354. package/src/nodes/sw-stats/publish-to.js +45 -0
  355. package/src/nodes/sw-stats/recompute.js +33 -0
  356. package/src/nodes/sw-stats/reset.js +15 -0
  357. package/src/nodes/sw-stats/update.js +39 -0
  358. package/src/nodes/swing-watch/index.js +17 -0
  359. package/src/nodes/swing-watch/init.js +144 -0
  360. package/src/nodes/swing-watch/introspect.js +188 -0
  361. package/src/nodes/swing-watch/publish-to.js +53 -0
  362. package/src/nodes/swing-watch/recompute.js +22 -0
  363. package/src/nodes/swing-watch/reset.js +55 -0
  364. package/src/nodes/swing-watch/update.js +767 -0
  365. package/src/nodes/tally/index.js +15 -0
  366. package/src/nodes/tally/init.js +32 -0
  367. package/src/nodes/tally/introspect.js +116 -0
  368. package/src/nodes/tally/publish-to.js +28 -0
  369. package/src/nodes/tally/recompute.js +10 -0
  370. package/src/nodes/tally/reset.js +10 -0
  371. package/src/nodes/tally/update.js +44 -0
  372. package/src/nodes/threshold/index.js +14 -0
  373. package/src/nodes/threshold/init.js +121 -0
  374. package/src/nodes/threshold/introspect.js +192 -0
  375. package/src/nodes/threshold/publish-to.js +11 -0
  376. package/src/nodes/threshold/recompute.js +8 -0
  377. package/src/nodes/threshold/reset.js +14 -0
  378. package/src/nodes/threshold/update.js +54 -0
  379. package/src/nodes/transform/helpers.js +39 -0
  380. package/src/nodes/transform/index.js +15 -0
  381. package/src/nodes/transform/init.js +35 -0
  382. package/src/nodes/transform/introspect.js +118 -0
  383. package/src/nodes/transform/publish-to.js +23 -0
  384. package/src/nodes/transform/recompute.js +12 -0
  385. package/src/nodes/transform/reset.js +11 -0
  386. package/src/nodes/transform/update.js +32 -0
  387. package/src/nodes/trend/compute-confidence.js +112 -0
  388. package/src/nodes/trend/index.js +14 -0
  389. package/src/nodes/trend/init.js +81 -0
  390. package/src/nodes/trend/introspect.js +151 -0
  391. package/src/nodes/trend/publish-to.js +22 -0
  392. package/src/nodes/trend/recompute.js +15 -0
  393. package/src/nodes/trend/reset.js +26 -0
  394. package/src/nodes/trend/update.js +93 -0
  395. package/src/nodes/tw-stats/copy-from.js +16 -0
  396. package/src/nodes/tw-stats/flush.js +12 -0
  397. package/src/nodes/tw-stats/index.js +41 -0
  398. package/src/nodes/tw-stats/init.js +92 -0
  399. package/src/nodes/tw-stats/introspect.js +171 -0
  400. package/src/nodes/tw-stats/publish-to.js +108 -0
  401. package/src/nodes/tw-stats/recompute.js +9 -0
  402. package/src/nodes/tw-stats/reset.js +16 -0
  403. package/src/nodes/tw-stats/update.js +113 -0
  404. package/src/nodes/unbalance/index.js +15 -0
  405. package/src/nodes/unbalance/init.js +59 -0
  406. package/src/nodes/unbalance/introspect.js +167 -0
  407. package/src/nodes/unbalance/publish-to.js +52 -0
  408. package/src/nodes/unbalance/recompute.js +11 -0
  409. package/src/nodes/unbalance/reset.js +10 -0
  410. package/src/nodes/unbalance/update.js +83 -0
  411. package/src/nodes/vector-distance/index.js +14 -0
  412. package/src/nodes/vector-distance/init.js +54 -0
  413. package/src/nodes/vector-distance/introspect.js +149 -0
  414. package/src/nodes/vector-distance/publish-to.js +35 -0
  415. package/src/nodes/vector-distance/recompute.js +11 -0
  416. package/src/nodes/vector-distance/reset.js +20 -0
  417. package/src/nodes/vector-distance/update.js +139 -0
  418. package/src/nodes/winnow/index.js +12 -0
  419. package/src/nodes/winnow/init.js +82 -0
  420. package/src/nodes/winnow/introspect.js +253 -0
  421. package/src/nodes/winnow/publish-to.js +51 -0
  422. package/src/nodes/winnow/recompute.js +14 -0
  423. package/src/nodes/winnow/reset.js +34 -0
  424. package/src/nodes/winnow/update.js +203 -0
  425. package/src/tools/training/features.js +227 -0
  426. package/src/tools/training/index.js +21 -0
  427. package/src/tools/training/logistic.js +238 -0
  428. package/src/tools/training/metrics.js +126 -0
  429. package/src/tools/training/scale.js +239 -0
  430. package/src/tools/training/validate.js +103 -0
  431. package/src/windowing/count-sliding/create.js +16 -0
  432. package/src/windowing/count-sliding/get-reverse-chronological-index.js +19 -0
  433. package/src/windowing/count-sliding/index.js +6 -0
  434. package/src/windowing/count-sliding/is-not-full.js +5 -0
  435. package/src/windowing/count-sliding/push.js +15 -0
  436. package/src/windowing/count-sliding/reset.js +11 -0
  437. package/src/windowing/count-sliding/size.js +5 -0
  438. package/src/windowing/time-sliding/create.js +25 -0
  439. package/src/windowing/time-sliding/push.js +42 -0
  440. package/CODE_OF_CONDUCT.md +0 -46
  441. package/CONTRIBUTING.md +0 -132
  442. package/docs/.nojekyll +0 -0
  443. package/docs/CONTRIBUTING.md +0 -134
  444. package/docs/README.md +0 -132
  445. package/docs/_coverpage.md +0 -26
  446. package/docs/_media/bg.png +0 -0
  447. package/docs/_media/wink-logo.png +0 -0
  448. package/docs/_navbar.md +0 -14
  449. package/docs/_sidebar.md +0 -2
  450. package/docs/index.html +0 -44
  451. package/eslint.config.js +0 -283
@@ -0,0 +1,1251 @@
1
+ // core/storage-manager/questdb/index.js
2
+
3
+ /**
4
+ * @fileoverview QuestDB storage adapter for time-series data persistence.
5
+ *
6
+ * Creates storage interface that integrates with partition-manager:
7
+ * - ensureTables: Creates tables via PostgreSQL wire protocol at startup
8
+ * - write(insightType, msg, partitionId): Writes row via ILP, sync return
9
+ * per the ADR-018 sink contract — `{ ok: true }` on success,
10
+ * `{ ok: false, error: { code, message } }` on failure.
11
+ * - flush(): Manual flush (in manual mode) or no-op (in auto mode)
12
+ * - shutdown(): Graceful shutdown
13
+ * - getPressure(): Buffer fill ratio in [0, 1] per ADR-018 (sync, O(1),
14
+ * allocation-free).
15
+ *
16
+ * Two flush modes:
17
+ * - auto: QuestDB client handles flushing based on rows/interval
18
+ * - manual: Application controls flush timing via idle timer
19
+ *
20
+ * Adapter contract (ADR-018, structured-sink role):
21
+ * - Module-level exports per the ADR-018 module surface: `id`, `configSchema`,
22
+ * `createStorage`, `durabilityClass`, `semanticsRequirement`, and the
23
+ * default aggregate (`questdbAdapter`) referencing the same constants.
24
+ * - Durability class (ADR-018): `'in-memory'` — rows accepted by `write()`
25
+ * live in the ILP client's in-process buffer until a flush reaches the
26
+ * server.
27
+ * A process crash loses every un-flushed row; the shutdown drain and
28
+ * its classified loss reporting exist precisely to shrink that window.
29
+ * - Hot-path `write` returns sync per ADR-013; the partition-manager never
30
+ * awaits a Promise on the per-message path.
31
+ * - `getPressure()` returns `(bufferedRows + inFlightRows) / autoFlushRows`,
32
+ * clamped to 1.0. `bufferedRows` counts rows awaiting the next flush and
33
+ * increments synchronously when the persist plan opens a row; a flush call
34
+ * moves its rows to `inFlightRows` (the client copies them out of the
35
+ * buffer synchronously — copy-out), where they stay until the flush
36
+ * settles. In auto mode the counter mirrors QuestDB's internal flush
37
+ * trigger via a heuristic reset at the `autoFlushRows` boundary; the
38
+ * `auto_flush_interval` trigger is not directly observable but self-heals
39
+ * via `checkIdleFlush` after `idleFlushAfterMs` of write-idle.
40
+ *
41
+ * Health-status semantics (uniform across sinks):
42
+ * - `getHealth()` returns `{status, connected, pressure, ...}` — the ADR-018
43
+ * health floor.
44
+ * - `connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD`.
45
+ * QuestDB has no persistent connection state to inspect (ILP sender is
46
+ * fire-and-forget); `connected` is *derived* from recent write success.
47
+ * - Status derivation:
48
+ * `red` if `!connected` (transport unhealthy or shutting down)
49
+ * `yellow` if `pressure >= 0.66` (threshold shared with ADR-020's
50
+ * Draft yield proposal) OR
51
+ * `consecutiveWriteErrors >= 1` (any error elevates concern)
52
+ * `green` otherwise
53
+ * - The counter resets to 0 on every successful write, so a single recovery
54
+ * write returns `status` to `green`.
55
+ *
56
+ * `err.code` vocabulary (per-adapter; ADR-018 has each adapter document
57
+ * its own codes in this header):
58
+ *
59
+ * Setup-time throws (ADR-018 fail-fast setup):
60
+ * - `INVALID_CONFIG` — the supplied configuration does not
61
+ * work. Five current sub-cases:
62
+ * (a) required transport URL missing (`ilpUrl`, `pgUrl`);
63
+ * (b) PostgreSQL endpoint answered but rejected the connection
64
+ * — wrong credentials or a protocol-level refusal;
65
+ * (c) a column declares an unsupported `type`, or a `float64`
66
+ * column declares a non-positive `resolution` (see
67
+ * `assert-columns.js`);
68
+ * (d) a table or column name the client's own ILP rules reject.
69
+ * Checked at plan build by driving each name through a
70
+ * throwaway client buffer (`assertIlpNames` in
71
+ * `persist-plan.js`) — a bad name fails deployment instead
72
+ * of wedging the sender mid-row at runtime;
73
+ * (e) an insightType uses the reserved column name `assetId`.
74
+ * Composer writes that column itself, from the partition id.
75
+ * Checked at plan build in `persist-plan.js`.
76
+ * Operator remediation: fix the supplied config or the relevant
77
+ * `QUESTDB_*` env var. The underlying transport error (sub-case b)
78
+ * is preserved on `err.cause` for diagnostics.
79
+ * - `TRANSPORT_UNREACHABLE` — the PostgreSQL endpoint did not answer
80
+ * at setup: nothing listening on the port, host unresolvable, no
81
+ * route, or the attempt timed out (a Node syscall code —
82
+ * `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, ...). Distinct from
83
+ * `INVALID_CONFIG` per the one split the ADR-018 error vocabulary
84
+ * mandates: the
85
+ * connection string may be fine — check the network, the firewall,
86
+ * whether QuestDB is running. Underlying error on `err.cause`.
87
+ * - `MISSING_ASSET_CLASS` — `assetClass` not provided to `createStorage`.
88
+ * Distinct from `INVALID_CONFIG` because operator remediation differs:
89
+ * they need to add `.assetClass(assetClassDef)` to the flow, not edit env
90
+ * vars or storage config.
91
+ * - `SCHEMA_ERROR` — DDL `CREATE TABLE` failed for a reason other
92
+ * than already-exists (see `ensure-tables.js`).
93
+ *
94
+ * Runtime returns:
95
+ * - `INVALID_INSIGHT_TYPE` — sync; the insightType has no persist plan
96
+ * (typically a config typo or asset-class drift).
97
+ * - `SEND_FAILED` — sync; the persist plan threw while invoking
98
+ * sender methods (ILP buffer overflow, type coercion failure, etc.).
99
+ * - `INVALID_TIMESTAMP` — reserved; today persist-plan handles invalid
100
+ * timestamps via `onWarning + skip-row`. It may later be surfaced as a
101
+ * hard return error.
102
+ *
103
+ * Shutdown-time throws (ADR-018 — a clean shutdown resolve means
104
+ * everything buffered or in flight was delivered; both carry
105
+ * `dropped: { count }`, exact). Shutdown settles every unsettled flush
106
+ * (idle, recovery) plus one final flush, raced against `{ timeout }`;
107
+ * its outcome is latched so repeated calls cannot contradict it:
108
+ * - `DELIVERY_FAILED` — a flush failed; first flush error on `cause`.
109
+ * - `SHUTDOWN_TIMEOUT` — delivery did not settle within the caller's
110
+ * `{ timeout }` (a send against an unreachable server never settles —
111
+ * see the mid-row recovery section below).
112
+ *
113
+ * Column-internal facts consumed (the ADR-018 column-internal facts
114
+ * pattern):
115
+ *
116
+ * QuestDB's top-level dependency on the asset class is declared in the
117
+ * `semanticsRequirement` export below: `name`, `columns`, `insightTypes`.
118
+ * Inside `columns`, QuestDB additionally reads two fields per column:
119
+ *
120
+ * - `columns.*.type` — read by `ensure-tables.js` (DDL type mapping
121
+ * `float64 → DOUBLE`, `int64 → LONG`, etc.) and by `persist-plan.js`
122
+ * (writer dispatch + numeric/non-numeric branch). Asserted at startup
123
+ * by `assert-columns.js`: must be present and a key in `DDL_TYPES`;
124
+ * throws `INVALID_CONFIG` otherwise.
125
+ * - `columns.*.resolution` — read by `persist-plan.js` (passed to the
126
+ * resolution-aware float64 writer factory in `writers.js`). Asserted
127
+ * at startup by `assert-columns.js`: when present on a `float64`
128
+ * column, must be a positive finite number; absent is fine
129
+ * (passthrough). Throws `INVALID_CONFIG` otherwise.
130
+ *
131
+ * Future column-internal reads (e.g., `unit`, `physicalRange`) would
132
+ * extend this section and add corresponding assertions in
133
+ * `assert-columns.js`.
134
+ *
135
+ * Async-rejection containment:
136
+ * - QuestDB's `sender.at(...)` is `async` — the buffer mutation is sync but
137
+ * the trailing `await this.tryFlush()` can fire a network flush. To prevent
138
+ * silent loss when that flush fails, `persist-plan.js` attaches a `.catch()`
139
+ * to the `at()` Promise and routes the failure to `onDeliveryFailure` when
140
+ * provided — otherwise it throws a classified `DELIVERY_FAILED` so the loss
141
+ * is loud. See `persist-plan.js` for the wrapper.
142
+ *
143
+ * Mid-row recovery (ADR-018 — a rejected message costs that message,
144
+ * nothing else):
145
+ * - A persist plan that throws between sender.table() and sender.at() leaves
146
+ * the client holding a half-written row. Without recovery, every later
147
+ * write fails with "Table name has already been set" — the 2026-06-10
148
+ * silent-write-failure incident lost 98.6% of a replay's rows this way.
149
+ * - write()'s catch therefore calls recoverSender(), which composes two
150
+ * documented client calls: flush() ships every COMPLETED row out of the
151
+ * buffer (the copy-out happens synchronously, before any network I/O; the
152
+ * client documents that an unfinished row stays behind), then reset()
153
+ * clears the buffer — at that point holding only the broken stub — and
154
+ * lowers the client's row-in-progress flags.
155
+ * - The client (4.2.0) has no row-cancel API, while its sibling clients do
156
+ * (.NET CancelRow, Rust/C rewind_to_marker, Java recovers automatically).
157
+ * Upstream issue #60 tracks the gap:
158
+ * https://github.com/questdb/nodejs-questdb-client/issues/60
159
+ * When a release ships cancelRow(), recoverSender() becomes that one call.
160
+ *
161
+ * Integration with persist-if node:
162
+ * - persist-if calls: state.storage.write(insightType, msg, partitionId)
163
+ * - This maps to: persistPlans[insightType](sender, msg, partitionId)
164
+ * - Reads `result.ok`, `result.error.code`, `result.error.message` per ADR-018.
165
+ *
166
+ * Storage Adapter Interface:
167
+ * - Exports `questdbAdapter` with `id` and `createStorage` for flow DSL
168
+ * - `createStorage(config)` is async, handles table creation and ILP connection
169
+ *
170
+ * Reading QuestDB rows back via PostgreSQL — the timezone gotcha:
171
+ *
172
+ * QuestDB stores TIMESTAMP columns as microseconds since the Unix epoch
173
+ * (no timezone). When you read those columns over the PostgreSQL wire
174
+ * protocol, QuestDB reports them as `TIMESTAMP without time zone`. The
175
+ * `pg` Node.js library treats that type as local-time and silently
176
+ * shifts the value by the local UTC offset when constructing a Date —
177
+ * so a non-UTC client sees a wrong `Date.getTime()`.
178
+ *
179
+ * The simplest fix: cast the column in QuestDB itself so pg never gets
180
+ * a chance to interpret it. `SELECT timestamp::long FROM ...` returns
181
+ * bigint microseconds, untouched. Divide by 1000 to recover ms.
182
+ *
183
+ * Example (Node.js with `pg`):
184
+ * SELECT _harnessId, timestamp::long AS ts_us, value FROM samples
185
+ * Then in JS: `tsMs = parseInt(row.ts_us, 10) / 1000;`
186
+ *
187
+ * Tests in non-UTC timezones almost always need this cast. See
188
+ * `src/core/source-manager/test-harness/test/e2e-contract-harness.specs.js`
189
+ * for a working example.
190
+ *
191
+ * @see https://questdb.com/docs/clients/ingest-node/
192
+ * @see docs/architecture/storage-layer.md
193
+ * @see ADR-018
194
+ */
195
+
196
+ import { Sender } from '@questdb/nodejs-client';
197
+ import pg from 'pg';
198
+
199
+ import { ENV_VARS } from '../../env-vars.js';
200
+ import { validators } from '../../utils/validate/index.js';
201
+ import { buildPersistPlans } from './persist-plan.js';
202
+ import { ensureTables } from './ensure-tables.js';
203
+ import { assertColumnFacts } from './assert-columns.js';
204
+
205
+ // ============================================================================
206
+ // HOT-PATH SINGLETONS
207
+ // ============================================================================
208
+
209
+ /**
210
+ * Singleton success result reused on every successful write. Hot-path zero
211
+ * allocation per ADR-013 / ADR-004. Plain literal — not frozen (V8 hot paths
212
+ * handle plain objects more predictably; no caller mutates this).
213
+ * @type {{ok: true}}
214
+ */
215
+ const RESULT_OK = { ok: true };
216
+
217
+ // Error results (INVALID_INSIGHT_TYPE, SEND_FAILED) are constructed per-call
218
+ // because each carries dynamic content (the offending insightType name and the
219
+ // underlying err.message respectively). These paths are rare; per-occurrence
220
+ // allocation on errors is acceptable. The RESULT_OK singleton above covers the
221
+ // per-message hot path.
222
+
223
+ /**
224
+ * Health status thresholds. consecutiveWriteErrors crossing the YELLOW
225
+ * threshold elevates status to 'yellow'; crossing the RED threshold flips
226
+ * `connected` to false (and therefore `status` to 'red'). Tuned for "any
227
+ * error is worth flagging" + "sustained errors mean the transport is gone."
228
+ */
229
+ const HEALTH_ERROR_YELLOW_THRESHOLD = 1;
230
+ const HEALTH_ERROR_RED_THRESHOLD = 5;
231
+
232
+ /**
233
+ * Pressure threshold above which `status` elevates to at least 'yellow'.
234
+ * The value matches the pressure-aware-yield design (ADR-020, still a
235
+ * Draft). That design proposes 0.66 as the pressure level where the
236
+ * flow would start yielding to let sinks drain; the number still needs
237
+ * benchmark confirmation. Today the yield trigger is time-only (ADR-024), so
238
+ * this alignment is forward-looking, not a description of current yield
239
+ * behaviour. Numeric — the constant exists once here rather than scattered
240
+ * as a magic number.
241
+ */
242
+ const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
243
+
244
+ // ============================================================================
245
+ // CONFIGURATION BUILDER
246
+ // ============================================================================
247
+
248
+ /**
249
+ * Build QuestDB Sender configuration from options.
250
+ *
251
+ * @param {Object} options - Storage options
252
+ * @param {string} options.ilpUrl - ILP endpoint (host:port)
253
+ * @param {string} options.flushMode - 'auto' or 'manual'
254
+ * @param {number} [options.autoFlushRows] - Rows before auto-flush
255
+ * @param {number} [options.autoFlushIntervalMs] - Time interval for auto-flush
256
+ * @param {number} [options.maxBufSize] - Maximum buffer size
257
+ * @param {number} [options.retryTimeout] - Retry timeout in ms
258
+ * @returns {string} Configuration string for Sender.fromConfig()
259
+ */
260
+ const buildSenderConfig = function ( options ) {
261
+ const { ilpUrl, flushMode, autoFlushRows, autoFlushIntervalMs, maxBufSize, retryTimeout } = options;
262
+
263
+ // Base configuration: HTTP transport
264
+ let config = `http::addr=${ilpUrl};`;
265
+
266
+ // Auto-flush configuration
267
+ if ( flushMode === 'auto' ) {
268
+ // Use QuestDB's built-in auto-flush
269
+ if ( autoFlushRows !== undefined ) {
270
+ config += `auto_flush_rows=${autoFlushRows};`;
271
+ }
272
+ if ( autoFlushIntervalMs !== undefined ) {
273
+ config += `auto_flush_interval=${autoFlushIntervalMs};`;
274
+ }
275
+ } else {
276
+ // Manual mode: disable auto-flush, we control timing
277
+ config += 'auto_flush=off;';
278
+ }
279
+
280
+ // Optional buffer/retry settings
281
+ if ( maxBufSize !== undefined ) {
282
+ config += `init_buf_size=${maxBufSize};`;
283
+ }
284
+ if ( retryTimeout !== undefined ) {
285
+ config += `retry_timeout=${retryTimeout};`;
286
+ }
287
+
288
+ return config;
289
+ };
290
+
291
+ // ============================================================================
292
+ // STORAGE FACTORY
293
+ // ============================================================================
294
+
295
+ /**
296
+ * Race a final flush against the shutdown time budget. No budget
297
+ * (0/undefined) means no enforcement — the await is unbounded, preserving
298
+ * direct-caller behavior. On timeout the flush promise is deliberately
299
+ * left pending: the client's send may never settle (see the file header),
300
+ * and the process is shutting down anyway. Its eventual rejection, if
301
+ * any, is absorbed by the race's own handlers — never an
302
+ * unhandledRejection.
303
+ *
304
+ * @param {Promise} flushPromise - the in-flight sender.flush()
305
+ * @param {number} timeoutMs - budget in ms; 0/absent disables the race
306
+ * @returns {Promise} settles with the flush, or rejects SHUTDOWN_TIMEOUT
307
+ */
308
+ const raceFlushTimeout = function ( flushPromise, timeoutMs ) {
309
+ if ( !( timeoutMs > 0 ) ) {
310
+ return flushPromise;
311
+ }
312
+ return new Promise( function ( resolve, reject ) {
313
+ const timer = setTimeout( function () {
314
+ const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
315
+ err.code = 'SHUTDOWN_TIMEOUT';
316
+ reject( err );
317
+ }, timeoutMs );
318
+ timer.unref();
319
+ flushPromise.then(
320
+ function ( value ) {
321
+ clearTimeout( timer );
322
+ resolve( value );
323
+ },
324
+
325
+ /* c8 ignore start -- unreachable from the sole caller:
326
+ doShutdown races a Promise.all over waits that absorb
327
+ their own rejections (that is what keeps the dropped
328
+ count exact), so the raced promise cannot reject today.
329
+ The handler stays because this utility's contract is
330
+ generic — without it, a rejecting promise from a future
331
+ caller would become an unhandled rejection and a race
332
+ that never settles. */
333
+ function ( err ) {
334
+ clearTimeout( timer );
335
+ reject( err );
336
+ }
337
+
338
+ /* c8 ignore stop */
339
+ );
340
+ } );
341
+ }; // raceFlushTimeout()
342
+
343
+ /**
344
+ * Node syscall codes that mean the PostgreSQL endpoint did not answer.
345
+ * Used at setup to classify a connect failure as TRANSPORT_UNREACHABLE
346
+ * (the one split the ADR-018 error vocabulary mandates — see the
347
+ * connect wrap below).
348
+ * Module-level Set: allocated once at load, membership check at setup.
349
+ *
350
+ * @type {Set<string>}
351
+ */
352
+ const NETWORK_ERROR_CODES = new Set( [
353
+ 'ECONNREFUSED',
354
+ 'ENOTFOUND',
355
+ 'ETIMEDOUT',
356
+ 'EHOSTUNREACH',
357
+ 'ENETUNREACH',
358
+ 'ECONNRESET',
359
+ 'EAI_AGAIN',
360
+ 'EPIPE'
361
+ ] );
362
+
363
+ /**
364
+ * Create QuestDB storage adapter.
365
+ *
366
+ * @param {Object} assetClass - Asset class definition with columns and insightTypes
367
+ * @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
368
+ * @param {Object} options - Configuration options
369
+ * @param {string} options.ilpUrl - ILP endpoint (host:port)
370
+ * @param {string} options.pgUrl - PostgreSQL endpoint (host:port)
371
+ * @param {string} [options.flushMode='auto'] - 'auto' or 'manual'
372
+ * @param {number} [options.idleFlushAfterMs=5000] - Idle time before flush (manual mode)
373
+ * @param {number} [options.idleFlushCheckMs=1000] - Idle check interval (manual mode)
374
+ * @param {number} [options.autoFlushRows] - Rows before auto-flush (auto mode)
375
+ * @param {number} [options.autoFlushIntervalMs] - Time interval for auto-flush (auto mode)
376
+ * @param {number} [options.maxBufSize] - Maximum buffer size
377
+ * @param {number} [options.retryTimeout] - Retry timeout in ms
378
+ * @param {string} [options.partitionBy='DAY'] - Table partition interval
379
+ * @param {function} [options.onWarning] - Warning callback for null column values
380
+ * @param {Object} [deps={}] - Injectable dependencies (for testing)
381
+ * @param {Object} [deps.SenderClass] - QuestDB Sender class (default: @questdb/nodejs-client Sender)
382
+ * @param {Object} [deps.PgClientClass] - PostgreSQL Client class (default: pg.Client)
383
+ * @returns {Promise<Object>} Storage adapter with write, flush, close methods
384
+ */
385
+ const createQuestDBStorage = async function ( assetClass, tablePrefix, options, deps = {} ) {
386
+ const {
387
+ ilpUrl = ENV_VARS.questdbIlpUrl,
388
+ pgUrl = ENV_VARS.questdbPgUrl,
389
+ flushMode = ENV_VARS.questdbFlushMode,
390
+ idleFlushAfterMs = ENV_VARS.questdbIdleFlushAfterMs,
391
+ idleFlushCheckMs = ENV_VARS.questdbIdleFlushCheckMs,
392
+ autoFlushRows = ENV_VARS.questdbAutoFlushRows,
393
+ autoFlushIntervalMs = ENV_VARS.questdbAutoFlushIntervalMs,
394
+ maxBufSize = ENV_VARS.questdbMaxBufSize,
395
+ retryTimeout = ENV_VARS.questdbRetryTimeout,
396
+ partitionBy = 'DAY',
397
+ onWarning,
398
+ onDeliveryFailure
399
+ } = options;
400
+
401
+ // Runtime validation — required from either DSL config or ENV_VARS.
402
+ // Per ADR-018, setup-time throws carry classified err.code.
403
+ if ( !ilpUrl ) {
404
+ const err = new Error( 'WinkComposer/questdb: ilpUrl required — set in .storage() config or QUESTDB_ILP_URL env var' );
405
+ err.code = 'INVALID_CONFIG';
406
+ throw err;
407
+ }
408
+ if ( !pgUrl ) {
409
+ const err = new Error( 'WinkComposer/questdb: pgUrl required — set in .storage() config or QUESTDB_PG_URL env var' );
410
+ err.code = 'INVALID_CONFIG';
411
+ throw err;
412
+ }
413
+
414
+ // Injectable dependencies with defaults
415
+ const {
416
+ SenderClass = Sender,
417
+ PgClientClass = pg.Client
418
+ } = deps;
419
+
420
+ // Build persist plans (pre-compiled closures)
421
+ const persistPlans = buildPersistPlans( assetClass, tablePrefix, { onWarning, onDeliveryFailure } );
422
+
423
+ // Ensure tables exist via PostgreSQL wire protocol
424
+ const [ pgHost, pgPort ] = pgUrl.split( ':' );
425
+ const pgClient = new PgClientClass( {
426
+ host: pgHost,
427
+ port: parseInt( pgPort, 10 ),
428
+ database: ENV_VARS.questdbDatabase,
429
+ user: ENV_VARS.questdbUser,
430
+ password: ENV_VARS.questdbPassword
431
+ } );
432
+
433
+ // Wrap the PostgreSQL connect so the failure carries a classified
434
+ // `err.code` an operator can route on. The ADR-018 error vocabulary
435
+ // mandates one split here: an endpoint that does not answer (a Node syscall
436
+ // code like ECONNREFUSED — see NETWORK_ERROR_CODES above) is
437
+ // TRANSPORT_UNREACHABLE, because the remediation is "check the
438
+ // network, the firewall, whether the service is running" — the
439
+ // connection string may be perfectly fine. Everything else — an
440
+ // auth failure (the host answered), a protocol error, an
441
+ // unclassified throw — stays INVALID_CONFIG: fix the supplied
442
+ // config. The underlying error is preserved via `err.cause` for
443
+ // diagnostics either way.
444
+ try {
445
+ await pgClient.connect();
446
+ } catch ( connErr ) {
447
+ const err = new Error(
448
+ `WinkComposer/questdb: could not connect to PostgreSQL at ${pgUrl} — ${connErr.message}`
449
+ );
450
+ err.code = NETWORK_ERROR_CODES.has( connErr.code ) ?
451
+ 'TRANSPORT_UNREACHABLE' :
452
+ 'INVALID_CONFIG';
453
+ err.cause = connErr;
454
+ throw err;
455
+ }
456
+
457
+ try {
458
+ await ensureTables( pgClient, assetClass, tablePrefix, { partitionBy } );
459
+ } finally {
460
+ await pgClient.end();
461
+ }
462
+
463
+ // Create ILP sender
464
+ const senderConfig = buildSenderConfig( {
465
+ ilpUrl,
466
+ flushMode,
467
+ autoFlushRows,
468
+ autoFlushIntervalMs,
469
+ maxBufSize,
470
+ retryTimeout
471
+ } );
472
+
473
+ // fromConfig returns a Promise that resolves to a connected sender
474
+ const sender = await SenderClass.fromConfig( senderConfig );
475
+
476
+ // Buffer state — `bufferedRows` counts rows awaiting the NEXT flush.
477
+ // The client's flush() copies its rows out of the buffer synchronously
478
+ // (copy-out) before any network I/O, so the moment a flush is called
479
+ // its rows are no longer buffered — they move to the in-flight tally
480
+ // below. Conflating the two quantities in one counter let shutdown
481
+ // report clean over a hung idle flush.
482
+ let lastWriteTime = 0;
483
+ let idleFlushTimer = null;
484
+ let bufferedRows = 0;
485
+
486
+ // Rows inside flush copies that have not settled yet, plus the flush
487
+ // promises carrying them. getPressure() adds this to bufferedRows (a
488
+ // hung flush is undelivered data and must read as pressure), and
489
+ // shutdown() settles these entries instead of firing a blind flush
490
+ // at a buffer the copy-out already emptied.
491
+ let inFlightRows = 0;
492
+ const inFlightFlushes = new Set();
493
+
494
+ // Shutdown outcome, latched on the first call. A lossy shutdown's
495
+ // failed flush already emptied the buffer via copy-out, so a re-run
496
+ // would find nothing to flush and resolve clean — contradicting the
497
+ // recorded loss. Every caller gets the first call's promise instead.
498
+ let shutdownPromise = null;
499
+
500
+ // Health state — drives getHealth()'s status/connected derivation.
501
+ // shuttingDown flips true at the start of shutdown(); never resets.
502
+ // consecutiveWriteErrors increments on every write() catch and resets to 0
503
+ // on every successful persist plan call (recovery). Pure derived signal —
504
+ // no extra hot-path allocation; just two integer/boolean operations.
505
+ let shuttingDown = false;
506
+ let consecutiveWriteErrors = 0;
507
+
508
+ // One idle flush in flight at a time. Against an unreachable server a
509
+ // flush can never settle (the client's retry loop — see the mid-row
510
+ // recovery section in the file header); without this guard every check
511
+ // interval would start ANOTHER stuck flush. A hung flush leaves the
512
+ // flag set, the buffer keeps filling, and the condition surfaces as
513
+ // rising pressure in getHealth() — visible, not multiplied.
514
+ let idleFlushInFlight = false;
515
+
516
+ /**
517
+ * Registers a flush the moment it is called. Copy-out means the rows
518
+ * leave the buffer NOW, delivered or not, so the caller hands them
519
+ * over in the same breath. Settlement — either way — removes them
520
+ * from the in-flight tally: resolved means delivered; rejected means
521
+ * lost, and loss REPORTING stays at the call site (the idle flush
522
+ * logs or routes to onDeliveryFailure, recovery routes to
523
+ * onDeliveryFailure, shutdown throws classified).
524
+ *
525
+ * @param {Promise} flushPromise - the just-fired sender.flush()
526
+ * @param {number} rows - row count the flush copy carries
527
+ * @returns {{promise: Promise, rows: number}} the tracked entry
528
+ */
529
+ const trackFlush = function ( flushPromise, rows ) {
530
+ const entry = { promise: flushPromise, rows };
531
+ inFlightRows += rows;
532
+ inFlightFlushes.add( entry );
533
+ const settle = function () {
534
+ inFlightFlushes.delete( entry );
535
+ inFlightRows -= rows;
536
+ };
537
+ flushPromise.then( settle, settle );
538
+ return entry;
539
+ }; // trackFlush()
540
+
541
+ /**
542
+ * Check and flush if idle for idleFlushAfterMs.
543
+ *
544
+ * Runs in BOTH modes (see longer comment block at the setInterval call
545
+ * below). The rows move to the in-flight tally the moment the flush
546
+ * fires; a successful settle clears them, which also bounds the
547
+ * worst-case pressure-counter lag in auto mode (when QuestDB's silent
548
+ * `auto_flush_interval` trigger fires without us observing it, this
549
+ * is the self-healing point).
550
+ */
551
+ const checkIdleFlush = async function () {
552
+ if ( idleFlushInFlight || bufferedRows === 0 ) {
553
+ return;
554
+ }
555
+
556
+ const now = Date.now();
557
+ const idleTime = now - lastWriteTime;
558
+
559
+ if ( idleTime >= idleFlushAfterMs ) {
560
+ idleFlushInFlight = true;
561
+ const rows = bufferedRows;
562
+ bufferedRows = 0;
563
+ const entry = trackFlush( sender.flush(), rows );
564
+ try {
565
+ await entry.promise;
566
+ } catch ( err ) {
567
+ // Copy-out: the failed flush carried the rows with it —
568
+ // the buffer no longer holds them and a retry cannot
569
+ // resend them. They are lost, and this is the report
570
+ // (the settle handler already cleared them from
571
+ // pressure). Same routing policy as the recovery flush:
572
+ // the caller owns the response when it asked to.
573
+ if ( onDeliveryFailure ) {
574
+ onDeliveryFailure( err, { idleFlush: true, rowsLost: rows } );
575
+ } else {
576
+ console.error(
577
+ `WinkComposer/questdb: idle flush failed; ${rows} buffered row(s) lost: ${err.message}`
578
+ );
579
+ }
580
+ } finally {
581
+ idleFlushInFlight = false;
582
+ }
583
+ }
584
+ };
585
+
586
+ // Start idle flush timer as safety net for "data stopped flowing" case.
587
+ // QuestDB's auto_flush_interval only checks elapsed time when NEW data is added -
588
+ // it is NOT a real background timer. If no more data arrives, the buffer sits
589
+ // indefinitely. QuestDB docs recommend: "implement your own timer-based logic."
590
+ // See: https://py-questdb-client.readthedocs.io/en/stable/conf.html
591
+ //
592
+ // This timer runs in BOTH modes:
593
+ // - Auto mode: Safety net for when data stops flowing (QuestDB won't flush on its own)
594
+ // - Manual mode: Primary flush mechanism (QuestDB auto-flush is disabled)
595
+ idleFlushTimer = setInterval( checkIdleFlush, idleFlushCheckMs );
596
+ // Don't block process exit
597
+ idleFlushTimer.unref();
598
+
599
+ // ========================================================================
600
+ // STORAGE INTERFACE
601
+ // ========================================================================
602
+
603
+ /**
604
+ * Cancels a half-written ILP row after a mid-row throw, so the NEXT write
605
+ * starts clean (see "Mid-row recovery" in the file header for the full
606
+ * account and the upstream issue link).
607
+ *
608
+ * flush() copies every completed row out of the buffer synchronously and
609
+ * sends them in the background; the unfinished row stays behind. reset()
610
+ * then clears the buffer — only the broken stub remains at that point —
611
+ * and lowers the client's row-in-progress flags. Net effect: the broken
612
+ * row vanishes, every good row is on its way, the sender accepts the
613
+ * next write.
614
+ *
615
+ * The early flush carries real data. If its send ultimately fails (after
616
+ * the client's own retries), that loss is reported like any other
617
+ * delivery failure: routed to `onDeliveryFailure` when provided,
618
+ * otherwise thrown as a classified unhandled rejection — the same policy
619
+ * as persist-plan's sender.at() failure path. Never silent.
620
+ */
621
+ const recoverSender = function () {
622
+ try {
623
+ // flush() is an async function: it can never throw synchronously,
624
+ // and the rows it sends live in its own copy of the buffer.
625
+ // Tracked like every flush, so shutdown settles it and its
626
+ // rows stay visible as pressure until it settles.
627
+ const entry = trackFlush( sender.flush(), bufferedRows );
628
+ sender.reset();
629
+ bufferedRows = 0;
630
+
631
+ entry.promise.catch( ( flushErr ) => {
632
+ if ( onDeliveryFailure ) {
633
+ onDeliveryFailure( flushErr, { recovery: true } );
634
+ return;
635
+ }
636
+ const failure = new Error(
637
+ `WinkComposer/questdb: recovery flush failed after a mid-row write error: ${flushErr.message}. ` +
638
+ 'Completed rows in that batch were dropped. Provide an `onDeliveryFailure` callback ' +
639
+ 'in the storage config to handle these explicitly.'
640
+ );
641
+ failure.code = 'DELIVERY_FAILED';
642
+ failure.cause = flushErr;
643
+ throw failure;
644
+ } );
645
+ } catch ( recoveryErr ) {
646
+ // Defensive: with the 4.2.0 client neither call can throw here
647
+ // (flush is async, reset is trivial buffer bookkeeping). If a
648
+ // future client changes that, the sender may stay wedged — but
649
+ // write() must still return its classified result rather than
650
+ // throw (ADR-018: the hot path never throws), and the
651
+ // failure must be visible.
652
+ console.error( `WinkComposer/questdb: sender recovery failed: ${recoveryErr.message}` );
653
+ }
654
+ }; // recoverSender()
655
+
656
+ /**
657
+ * Write a message to QuestDB for a given insightType.
658
+ *
659
+ * Sync hot-path return per ADR-018 / ADR-013:
660
+ * - `RESULT_OK` (singleton) on successful enqueue.
661
+ * - `{ ok: false, error: { code: 'INVALID_INSIGHT_TYPE', message } }`
662
+ * when the insightType has no persist plan (config error).
663
+ * - `{ ok: false, error: { code: 'SEND_FAILED', message } }` when the
664
+ * persist plan throws (sender method failure: ILP buffer overflow,
665
+ * type coercion failure, etc.).
666
+ *
667
+ * Never throws. Async tryFlush rejections inside `sender.at()` are
668
+ * contained at the call site in `persist-plan.js` (see file header).
669
+ *
670
+ * @param {string} insightType - SignalType name (must exist in assetClass)
671
+ * @param {Object} message - Message with column values
672
+ * @param {string} partitionId - Partition identifier (stored as SYMBOL)
673
+ * @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
674
+ */
675
+ const write = function ( insightType, message, partitionId ) {
676
+ if ( shuttingDown ) {
677
+ // The idle timer is already stopped and the final flush may
678
+ // have run: a row accepted now would have no flusher left.
679
+ // Refuse loudly instead of stranding it — uniform with the
680
+ // MQTT emitter's SHUTTING_DOWN refusal.
681
+ return {
682
+ ok: false,
683
+ error: {
684
+ code: 'SHUTTING_DOWN',
685
+ message: 'WinkComposer/questdb: write rejected — storage is shutting down'
686
+ }
687
+ };
688
+ }
689
+
690
+ const persistPlan = persistPlans[ insightType ];
691
+
692
+ if ( !persistPlan ) {
693
+ return {
694
+ ok: false,
695
+ error: {
696
+ code: 'INVALID_INSIGHT_TYPE',
697
+ message: `No persist plan for insightType '${insightType}'`
698
+ }
699
+ };
700
+ }
701
+
702
+ try {
703
+ // The plan reports whether it actually opened a row. A row
704
+ // skipped in phase 1 (bad designated timestamp) never touched
705
+ // the sender and must not count as buffered — it would inflate
706
+ // pressure and shutdown's dropped count.
707
+ const written = persistPlan( sender, message, partitionId );
708
+ consecutiveWriteErrors = 0; // recovery — health flips back to green
709
+ if ( written ) {
710
+ lastWriteTime = Date.now();
711
+ bufferedRows += 1;
712
+ // Auto mode: QuestDB triggers an internal flush on the sender.at()
713
+ // call that brings pendingRowCount to autoFlushRows (verified
714
+ // against @questdb/nodejs-client v4.2.0 dist/cjs/index.js
715
+ // tryFlush: pendingRowCount >= autoFlushRows). Mirror that reset
716
+ // so getPressure() reflects QuestDB's actual buffer state.
717
+ //
718
+ // Guarded on `autoFlushRows > 0` because the value is optional in
719
+ // ENV_VARS / config; when unset, QuestDB falls back to its own
720
+ // internal default which we cannot observe. In that case the
721
+ // heuristic does not fire and QuestDB contributes no pressure
722
+ // signal — `getPressure()` (below) returns 0.
723
+ if ( flushMode === 'auto' && autoFlushRows > 0 && bufferedRows >= autoFlushRows ) {
724
+ bufferedRows = 0;
725
+ }
726
+ }
727
+ return RESULT_OK;
728
+ } catch ( err ) {
729
+ consecutiveWriteErrors += 1; // health degradation signal
730
+ // The throw may have left a half-written row in the sender;
731
+ // cancel it so this failure costs one row, not the rest of the
732
+ // run (ADR-018). Safe to call even when no row was open: flushing
733
+ // early is harmless and reset() on a consistent buffer is a no-op.
734
+ recoverSender();
735
+ return {
736
+ ok: false,
737
+ error: {
738
+ code: 'SEND_FAILED',
739
+ message: err.message
740
+ }
741
+ };
742
+ }
743
+ };
744
+
745
+ /**
746
+ * Flush pending rows to QuestDB.
747
+ * In auto mode, this is a no-op (QuestDB handles it).
748
+ * In manual mode, this forces an immediate flush. The rows move to
749
+ * the in-flight tally at the call (copy-out); a failure rejects to
750
+ * the caller exactly as before, and the settle handler keeps the
751
+ * pressure accounting straight either way.
752
+ *
753
+ * @returns {Promise<void>}
754
+ */
755
+ const flush = async function () {
756
+ if ( flushMode === 'manual' && bufferedRows > 0 ) {
757
+ const entry = trackFlush( sender.flush(), bufferedRows );
758
+ bufferedRows = 0;
759
+ await entry.promise;
760
+ }
761
+ // Auto mode: no-op, QuestDB handles flushing
762
+ };
763
+
764
+ /**
765
+ * Backpressure metric for the partition manager. Returns the buffer fill
766
+ * ratio in [0, 1] per ADR-018 (sync, O(1), allocation-free). Clamped
767
+ * to 1.0 — the codebase invariant for getPressure across adapters.
768
+ *
769
+ * Accuracy by mode:
770
+ * - Manual mode: exact (we own every flush).
771
+ * - Auto mode, heavy flow: exact (autoFlushRows heuristic reset matches).
772
+ * - Auto mode, slow flow + auto_flush_interval triggers: over-states by
773
+ * up to `idleFlushAfterMs`, self-heals via `checkIdleFlush`.
774
+ *
775
+ * Rows inside unsettled flush copies count as pressure: a hung flush
776
+ * is undelivered data, and hiding it is what let shutdown report
777
+ * clean over it. They leave the tally when their flush settles —
778
+ * delivered or reported lost.
779
+ *
780
+ * When `autoFlushRows` is not configured (left to QuestDB's internal
781
+ * default), we have no observable capacity reference and return
782
+ * 0 — QuestDB then contributes nothing to the flow's pressure signal.
783
+ * The time-based yield (ADR-024) and message-count safety nets remain
784
+ * effective.
785
+ *
786
+ * @returns {number} Pressure value in [0, 1]
787
+ */
788
+ const getPressure = function () {
789
+ if ( !( autoFlushRows > 0 ) ) {
790
+ return 0;
791
+ }
792
+ return Math.min( 1.0, ( bufferedRows + inFlightRows ) / autoFlushRows );
793
+ };
794
+
795
+ /**
796
+ * Health snapshot for operator monitoring (uniform across sinks).
797
+ * Returns the ADR-018 health floor `{status, connected, ...}`, plus a
798
+ * small set of QuestDB-specific diagnostic fields.
799
+ *
800
+ * Status derivation (kept in code, not config — operator mental model is
801
+ * load-bearing institutional knowledge):
802
+ * - `red` if `!connected` (shutting down or sustained write failure)
803
+ * - `yellow` if `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD` OR
804
+ * `consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD`
805
+ * - `green` otherwise
806
+ *
807
+ * `connected` here is *derived* — QuestDB's ILP sender is fire-and-forget
808
+ * with no observable socket state, so we infer transport health from
809
+ * recent write success.
810
+ *
811
+ * @returns {{status: 'green'|'yellow'|'red', connected: boolean, pressure: number, consecutiveWriteErrors: number, bufferedRows: number, flushMode: string}}
812
+ */
813
+ const getHealth = function () {
814
+ const pressure = getPressure();
815
+ const connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD;
816
+
817
+ let status;
818
+ if ( !connected ) {
819
+ status = 'red';
820
+ } else if ( pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD || consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD ) {
821
+ status = 'yellow';
822
+ } else {
823
+ status = 'green';
824
+ }
825
+
826
+ return {
827
+ // Required health floor (ADR-018)
828
+ status,
829
+ connected,
830
+ pressure,
831
+ // Adapter-specific diagnostics
832
+ consecutiveWriteErrors,
833
+ bufferedRows,
834
+ flushMode
835
+ };
836
+ };
837
+
838
+ /**
839
+ * Best-effort transport close on the lossy path: the loss report
840
+ * (the classified throw that follows) matters more than a close
841
+ * failure, which is only logged.
842
+ */
843
+ const closeQuietly = function () {
844
+ return sender.close().catch( function ( closeErr ) {
845
+ console.error( `WinkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
846
+ } );
847
+ }; // closeQuietly()
848
+
849
+ /**
850
+ * The real shutdown body. `shutdown` below latches its promise so
851
+ * every caller — including re-entrant and post-failure callers —
852
+ * receives this one outcome.
853
+ *
854
+ * A clean resolve is a delivery statement (ADR-018): everything
855
+ * buffered OR in flight was delivered. Shutdown therefore settles
856
+ * every unsettled flush (idle, recovery) plus one final flush for
857
+ * whatever is still buffered, all raced against the caller's
858
+ * `{ timeout }` (ADR-018 drain-then-close). It never fires a blind flush at a
859
+ * buffer an earlier copy-out emptied — that is what let it report
860
+ * clean over a hung idle flush.
861
+ *
862
+ * On loss it rejects classified, `dropped: { count }` exact:
863
+ * - any awaited flush fails → `DELIVERY_FAILED`, first flush error
864
+ * on `cause`, count = rows on the flushes that failed;
865
+ * - the combined wait does not settle in time → `SHUTDOWN_TIMEOUT`,
866
+ * count = rows not confirmed delivered. A send against an
867
+ * unreachable server never settles (the client's retry loop — see
868
+ * the file header), so the bound is what keeps shutdown finite.
869
+ * `dropped` is a statement about THIS session: those rows were not
870
+ * confirmed delivered before close. An abandoned flush keeps
871
+ * retrying and may still land its rows later if the server
872
+ * recovers — the count is a floor on uncertainty, not a proof of
873
+ * loss.
874
+ *
875
+ * The transport close is attempted in both loss paths, but on the
876
+ * HTTP transport the client's `close()` is an empty function
877
+ * (verified against @questdb/nodejs-client 4.2.0), so nothing can
878
+ * abort an abandoned flush's retry timers from here; they keep the
879
+ * event loop alive. Process exit is the final backstop — in a flow,
880
+ * the shutdown manager's `SHUTDOWN_FORCE_TIMEOUT_MS` exit covers
881
+ * this. No timeout supplied = no enforcement (unbounded await),
882
+ * preserving direct-caller behavior.
883
+ */
884
+ const doShutdown = async function ( timeout ) {
885
+ // Flip the health flag first so any concurrent getHealth() call
886
+ // immediately sees the shutdown and returns red/disconnected —
887
+ // and write() starts refusing new rows (SHUTTING_DOWN).
888
+ shuttingDown = true;
889
+
890
+ // Stop idle flush timer
891
+ if ( idleFlushTimer ) {
892
+ clearInterval( idleFlushTimer );
893
+ idleFlushTimer = null;
894
+ }
895
+
896
+ // Everything delivery still owes: flushes already in flight
897
+ // (their rows left the buffer at their call) plus one final
898
+ // flush for whatever is still buffered. Each wait records its
899
+ // outcome into the tallies below; the mapped promises never
900
+ // reject, so the only rejection the race can surface is the
901
+ // timeout itself.
902
+ let totalRows = 0;
903
+ let deliveredRows = 0;
904
+ let failedRows = 0;
905
+ let firstFailure = null;
906
+ const waits = [];
907
+
908
+ const awaitDelivery = function ( entry ) {
909
+ totalRows += entry.rows;
910
+ waits.push( entry.promise.then(
911
+ function () {
912
+ deliveredRows += entry.rows;
913
+ },
914
+ function ( err ) {
915
+ failedRows += entry.rows;
916
+ if ( !firstFailure ) {
917
+ firstFailure = err;
918
+ }
919
+ }
920
+ ) );
921
+ }; // awaitDelivery()
922
+
923
+ inFlightFlushes.forEach( awaitDelivery );
924
+ if ( bufferedRows > 0 ) {
925
+ const entry = trackFlush( sender.flush(), bufferedRows );
926
+ bufferedRows = 0;
927
+ awaitDelivery( entry );
928
+ }
929
+
930
+ if ( waits.length > 0 ) {
931
+ try {
932
+ await raceFlushTimeout( Promise.all( waits ), timeout );
933
+ } catch ( err ) {
934
+ await closeQuietly();
935
+ const dropped = totalRows - deliveredRows;
936
+ const timedOut = new Error(
937
+ `WinkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
938
+ );
939
+ timedOut.code = 'SHUTDOWN_TIMEOUT';
940
+ timedOut.dropped = { count: dropped };
941
+ throw timedOut;
942
+ }
943
+
944
+ if ( failedRows > 0 ) {
945
+ await closeQuietly();
946
+ const failure = new Error(
947
+ `WinkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
948
+ );
949
+ failure.code = 'DELIVERY_FAILED';
950
+ failure.dropped = { count: failedRows };
951
+ failure.cause = firstFailure;
952
+ throw failure;
953
+ }
954
+ }
955
+
956
+ // Close sender
957
+ await sender.close();
958
+ }; // doShutdown()
959
+
960
+ /**
961
+ * Shutdown the storage adapter gracefully.
962
+ * Flushes pending data and closes connections.
963
+ *
964
+ * Called by wire-storages.shutdown() which is invoked during:
965
+ * - Pipeline shutdown (flowHandle.shutdown())
966
+ * - Process signal handlers (SIGINT, SIGTERM)
967
+ *
968
+ * The outcome is latched: the first call runs the shutdown, every
969
+ * later call returns the same promise. A lossy shutdown's failed
970
+ * flush already emptied the buffer (copy-out), so a re-run would
971
+ * find nothing to flush and resolve clean, contradicting the
972
+ * recorded loss. One consequence: the first caller's `{ timeout }`
973
+ * governs; a later caller's is ignored.
974
+ *
975
+ * @param {{timeout?: number}} [options]
976
+ * @returns {Promise<void>}
977
+ */
978
+ const shutdown = function ( { timeout = 0 } = {} ) {
979
+ if ( !shutdownPromise ) {
980
+ shutdownPromise = doShutdown( timeout );
981
+ }
982
+ return shutdownPromise;
983
+ }; // shutdown()
984
+
985
+ return {
986
+ write,
987
+ flush,
988
+ shutdown,
989
+ getPressure,
990
+ getHealth,
991
+ // Expose for testing/debugging
992
+ _sender: sender,
993
+ _persistPlans: persistPlans
994
+ };
995
+ };
996
+
997
+ // ============================================================================
998
+ // CONFIG SCHEMA
999
+ // ============================================================================
1000
+
1001
+ /**
1002
+ * Configuration schema for QuestDB storage adapter validation.
1003
+ * Used by flow.storage() to validate config at DSL time.
1004
+ *
1005
+ * `_propertyNames` lists every accepted key — unknown keys throw at
1006
+ * DSL time (the validator's only unknown-key mechanism).
1007
+ * Two factory-visible keys are deliberately absent: `assetClass`
1008
+ * (wire-storages injects it from the flow's `.assetClass()` after DSL
1009
+ * validation, so a user-supplied value would be silently overwritten)
1010
+ * and `_deps` (direct-call test injection only; direct calls bypass
1011
+ * this schema entirely).
1012
+ *
1013
+ * @type {Object}
1014
+ */
1015
+ const configSchema = {
1016
+ _propertyNames: [
1017
+ 'ilpUrl',
1018
+ 'pgUrl',
1019
+ 'tablePrefix',
1020
+ 'flushMode',
1021
+ 'idleFlushAfterMs',
1022
+ 'idleFlushCheckMs',
1023
+ 'autoFlushRows',
1024
+ 'autoFlushIntervalMs',
1025
+ 'maxBufSize',
1026
+ 'retryTimeout',
1027
+ 'partitionBy',
1028
+ 'onWarning',
1029
+ 'onDeliveryFailure'
1030
+ ],
1031
+ ilpUrl: {
1032
+ type: 'string',
1033
+ required: false,
1034
+ minLength: 1,
1035
+ error: 'ilpUrl must be a non-empty string (e.g., localhost:9000)'
1036
+ },
1037
+ pgUrl: {
1038
+ type: 'string',
1039
+ required: false,
1040
+ minLength: 1,
1041
+ error: 'pgUrl must be a non-empty string (e.g., localhost:8812)'
1042
+ },
1043
+ tablePrefix: {
1044
+ type: 'string',
1045
+ required: false,
1046
+ minLength: 1,
1047
+ error: 'tablePrefix must be a non-empty string (defaults to assetClass.name when omitted)'
1048
+ },
1049
+ flushMode: {
1050
+ type: 'string',
1051
+ required: false,
1052
+ validator: validators.oneOf( [ 'auto', 'manual' ] ),
1053
+ error: 'flushMode must be "auto" or "manual"'
1054
+ },
1055
+ idleFlushAfterMs: {
1056
+ type: 'number',
1057
+ required: false,
1058
+ validator: validators.positiveInteger,
1059
+ error: 'idleFlushAfterMs must be a positive integer'
1060
+ },
1061
+ idleFlushCheckMs: {
1062
+ type: 'number',
1063
+ required: false,
1064
+ validator: validators.positiveInteger,
1065
+ error: 'idleFlushCheckMs must be a positive integer'
1066
+ },
1067
+ autoFlushRows: {
1068
+ type: 'number',
1069
+ required: false,
1070
+ validator: validators.positiveInteger,
1071
+ error: 'autoFlushRows must be a positive integer'
1072
+ },
1073
+ autoFlushIntervalMs: {
1074
+ type: 'number',
1075
+ required: false,
1076
+ validator: validators.positiveInteger,
1077
+ error: 'autoFlushIntervalMs must be a positive integer'
1078
+ },
1079
+ maxBufSize: {
1080
+ type: 'number',
1081
+ required: false,
1082
+ validator: validators.positiveInteger,
1083
+ error: 'maxBufSize must be a positive integer'
1084
+ },
1085
+ retryTimeout: {
1086
+ type: 'number',
1087
+ required: false,
1088
+ validator: validators.positiveInteger,
1089
+ error: 'retryTimeout must be a positive integer'
1090
+ },
1091
+ partitionBy: {
1092
+ type: 'string',
1093
+ required: false,
1094
+ validator: validators.oneOf( [ 'NONE', 'HOUR', 'DAY', 'WEEK', 'MONTH', 'YEAR' ] ),
1095
+ error: 'partitionBy must be one of: NONE, HOUR, DAY, WEEK, MONTH, YEAR'
1096
+ },
1097
+ onWarning: {
1098
+ type: 'function',
1099
+ required: false,
1100
+ error: 'onWarning must be a function'
1101
+ },
1102
+ onDeliveryFailure: {
1103
+ type: 'function',
1104
+ required: false,
1105
+ error: 'onDeliveryFailure must be a function'
1106
+ }
1107
+ };
1108
+
1109
+ // ============================================================================
1110
+ // ADAPTER EXPORT
1111
+ // ============================================================================
1112
+
1113
+ /**
1114
+ * Adapter identifier — matched by the flow registry and used as the
1115
+ * prefix in error messages.
1116
+ * @type {string}
1117
+ */
1118
+ const id = 'questdb';
1119
+
1120
+ /**
1121
+ * Crash-survival class per ADR-018. Rows accepted by `write()` sit in
1122
+ * the ILP client's in-process buffer until flushed; a crash loses every
1123
+ * un-flushed row. The declaration states exactly what the code does; a
1124
+ * local write-ahead log could raise the class later.
1125
+ * @type {string}
1126
+ */
1127
+ const durabilityClass = 'in-memory';
1128
+
1129
+ // Capability declaration per ADR-018.
1130
+ //
1131
+ // QuestDB needs the asset class because (a) ensure-tables.js reads
1132
+ // `columns` and `insightTypes` to issue CREATE TABLE statements, and
1133
+ // (b) persist-plan.js reads the same to dispatch column writers and
1134
+ // pre-compile resolution-aware quantization. `name` drives the default
1135
+ // `tablePrefix` when the caller did not pass one.
1136
+ //
1137
+ // Declared `required: true` because there is no useful behaviour for
1138
+ // QuestDB without an assetClass — the wiring layer fails fast with
1139
+ // MISSING_ASSET_CLASS rather than calling `createStorage` and letting
1140
+ // the failure surface deeper.
1141
+ //
1142
+ // `fields` lists exactly the top-level fields read. The slicing is
1143
+ // top-level only; column-internal fields (`type`, `resolution`) are
1144
+ // read directly from the `columns` slice and validated by Layer 2
1145
+ // assertions inside `createStorage` below. A declarative shape for
1146
+ // column-internal capability is deferred until a second adapter
1147
+ // needs it.
1148
+ const semanticsRequirement = {
1149
+ assetClass: {
1150
+ required: true,
1151
+ fields: [ 'name', 'columns', 'insightTypes' ]
1152
+ }
1153
+ };
1154
+
1155
+ /**
1156
+ * Create QuestDB storage instance.
1157
+ *
1158
+ * @param {Object} config - Configuration object
1159
+ * @param {Object} config.assetClass - Asset class definition with columns and insightTypes
1160
+ * @param {string} config.tablePrefix - Prefix for table names (defaults to assetClass.name)
1161
+ * @param {string} config.ilpUrl - ILP endpoint (host:port)
1162
+ * @param {string} config.pgUrl - PostgreSQL endpoint (host:port)
1163
+ * @param {string} [config.flushMode='auto'] - 'auto' or 'manual'
1164
+ * @param {number} [config.idleFlushAfterMs=5000] - Idle time before flush (manual mode)
1165
+ * @param {number} [config.idleFlushCheckMs=1000] - Idle check interval (manual mode)
1166
+ * @param {number} [config.autoFlushRows] - Rows before auto-flush (auto mode)
1167
+ * @param {number} [config.autoFlushIntervalMs] - Time interval for auto-flush (auto mode)
1168
+ * @param {number} [config.maxBufSize] - Maximum buffer size
1169
+ * @param {number} [config.retryTimeout] - Retry timeout in ms
1170
+ * @param {string} [config.partitionBy='DAY'] - Table partition interval
1171
+ * @param {function} [config.onWarning] - Warning callback for null column values
1172
+ * @param {Object} [config._deps] - Injectable dependencies (for testing)
1173
+ * @returns {Promise<Object>} Storage instance with write, flush, shutdown methods
1174
+ */
1175
+ const createStorage = function ( config ) {
1176
+ const { assetClass, tablePrefix, _deps, ...options } = config;
1177
+
1178
+ // Validate assetClass (injected by wire-storages.js from flow's .assetClass())
1179
+ // tablePrefix is auto-defaulted to assetClass.name by wire-storages.js
1180
+ if ( !assetClass ) {
1181
+ // Per ADR-018, setup-time throws carry classified err.code.
1182
+ // MISSING_ASSET_CLASS (not INVALID_CONFIG) because the operator
1183
+ // remediation differs — they need to add .assetClass() to the
1184
+ // flow definition, not edit env vars or storage config.
1185
+ const err = new Error(
1186
+ 'WinkComposer/questdb: assetClass is required - add .assetClass(assetClassDef) to flow before .storage()'
1187
+ );
1188
+ err.code = 'MISSING_ASSET_CLASS';
1189
+ throw err;
1190
+ }
1191
+
1192
+ // Adapter-side defensive validation of column-internal facts
1193
+ // QuestDB consumes. Per the ADR-018 column-internal facts
1194
+ // pattern: the universal semantics
1195
+ // schema validates top-level
1196
+ // structure (type required, type in COLUMN_TYPES) at load and
1197
+ // at .assetClass() time; this layer catches bypasses (e.g.,
1198
+ // direct createStorage call from test code) and adapter-specific
1199
+ // requirements like "float64 columns with declared resolution
1200
+ // must have a positive value." See `assert-columns.js` for the
1201
+ // full rationale.
1202
+ assertColumnFacts( assetClass );
1203
+
1204
+ // Default `tablePrefix` to the asset class name when the caller
1205
+ // did not supply one. This default used to live in wire-storages
1206
+ // and moved here so the wiring layer stays generic and each
1207
+ // adapter owns its own defaulting policy. Same observable
1208
+ // behaviour: callers passing an explicit `tablePrefix` see it
1209
+ // honoured; callers omitting it get `assetClass.name`.
1210
+ const effectiveTablePrefix = tablePrefix ?? assetClass.name;
1211
+
1212
+ // Returns Promise from createQuestDBStorage (already async)
1213
+ return createQuestDBStorage( assetClass, effectiveTablePrefix, options, _deps );
1214
+ }; // createStorage()
1215
+
1216
+ /**
1217
+ * QuestDB storage adapter for flow DSL integration — the default
1218
+ * aggregate ADR-018's module surface calls for. References the same
1219
+ * constants as the named exports; never
1220
+ * a second source of truth.
1221
+ *
1222
+ * Usage:
1223
+ * import questdbAdapter from './core/storage-manager/questdb/index.js';
1224
+ *
1225
+ * flow( 'myFlow' )
1226
+ * .assetClass( assetClassDef ) // wire-storages injects the slice
1227
+ * .storage( questdbAdapter, {
1228
+ * tablePrefix: 'myPrefix',
1229
+ * ilpUrl: 'localhost:9000',
1230
+ * pgUrl: 'localhost:8812'
1231
+ * } )
1232
+ *
1233
+ * @type {Object}
1234
+ */
1235
+ const questdbAdapter = { id, configSchema, durabilityClass, semanticsRequirement, createStorage };
1236
+
1237
+ // ============================================================================
1238
+ // EXPORTS
1239
+ // ============================================================================
1240
+
1241
+ export default questdbAdapter;
1242
+ export {
1243
+ id,
1244
+ configSchema,
1245
+ durabilityClass,
1246
+ semanticsRequirement,
1247
+ createStorage,
1248
+ createQuestDBStorage,
1249
+ buildSenderConfig,
1250
+ questdbAdapter
1251
+ };