@winkjs/composer 0.0.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +42 -103
- package/package.json +53 -29
- package/src/composer.js +75 -0
- package/src/core/batch/aoo-encoder.js +148 -0
- package/src/core/batch/index.js +12 -0
- package/src/core/batch/sliding-window.js +91 -0
- package/src/core/codec/index.js +137 -0
- package/src/core/emitter-manager/mqtt/constants.js +102 -0
- package/src/core/emitter-manager/mqtt/emitter.js +780 -0
- package/src/core/emitter-manager/mqtt/index.js +190 -0
- package/src/core/emitter-manager/mqtt/mqtt-store.js +538 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +98 -0
- package/src/core/emitter-manager/terminal/formatters.js +160 -0
- package/src/core/emitter-manager/terminal/index.js +341 -0
- package/src/core/env-vars.js +245 -0
- package/src/core/mqtt-protocol.js +47 -0
- package/src/core/partition-manager/index.js +7 -0
- package/src/core/partition-manager/init.js +44 -0
- package/src/core/partition-manager/introspect.js +15 -0
- package/src/core/partition-manager/publish-to.js +6 -0
- package/src/core/partition-manager/recompute.js +5 -0
- package/src/core/partition-manager/reset.js +5 -0
- package/src/core/partition-manager/update.js +264 -0
- package/src/core/partition-manager/validate-flow.js +157 -0
- package/src/core/semantics/digest.js +128 -0
- package/src/core/semantics/index.js +83 -0
- package/src/core/semantics/loader.js +571 -0
- package/src/core/semantics/schemas/asset-class-schema.js +218 -0
- package/src/core/semantics/schemas/column-schema.js +651 -0
- package/src/core/semantics/schemas/enum-schema.js +153 -0
- package/src/core/semantics/schemas/index.js +51 -0
- package/src/core/semantics/warnings.js +141 -0
- package/src/core/shutdown-manager/index.js +289 -0
- package/src/core/source-manager/csv/index.js +171 -0
- package/src/core/source-manager/csv/start.js +456 -0
- package/src/core/source-manager/mqtt/client.js +406 -0
- package/src/core/source-manager/mqtt/constants.js +147 -0
- package/src/core/source-manager/mqtt/dedup.js +244 -0
- package/src/core/source-manager/mqtt/index.js +260 -0
- package/src/core/source-manager/mqtt/status.js +546 -0
- package/src/core/source-manager/test-harness/comparator.js +226 -0
- package/src/core/source-manager/test-harness/field-generator.js +85 -0
- package/src/core/source-manager/test-harness/fuzz.js +87 -0
- package/src/core/source-manager/test-harness/index.js +115 -0
- package/src/core/source-manager/test-harness/prng.js +57 -0
- package/src/core/source-manager/test-harness/start.js +232 -0
- package/src/core/source-manager/test-harness/validate.js +213 -0
- package/src/core/storage-manager/questdb/assert-columns.js +120 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +187 -0
- package/src/core/storage-manager/questdb/index.js +1251 -0
- package/src/core/storage-manager/questdb/persist-plan.js +488 -0
- package/src/core/storage-manager/questdb/writers.js +142 -0
- package/src/core/test-utils/sealed-record.js +80 -0
- package/src/core/test-utils/soak-signature.js +149 -0
- package/src/core/test-utils/tcp-proxy.js +106 -0
- package/src/core/tunable/helpers.js +254 -0
- package/src/core/tunable/index.js +87 -0
- package/src/core/utils/console-formatter/index.js +207 -0
- package/src/core/utils/create-smart-store-as/index.js +98 -0
- package/src/core/utils/create-smart-store-as/sanitize-field.js +46 -0
- package/src/core/utils/create-smart-store-as/smart-truncate.js +93 -0
- package/src/core/utils/flow/index.js +5 -0
- package/src/core/utils/flow/make-collision-checker.js +23 -0
- package/src/core/utils/flow/to-kebab.js +20 -0
- package/src/core/utils/half-life/index.js +111 -0
- package/src/core/utils/node/disable.js +11 -0
- package/src/core/utils/node/enable.js +11 -0
- package/src/core/utils/node/execute-triggers.js +40 -0
- package/src/core/utils/node/index.js +10 -0
- package/src/core/utils/node/pause.js +11 -0
- package/src/core/utils/node/populate-predicate-inputs.js +14 -0
- package/src/core/utils/node/publish-nan.js +19 -0
- package/src/core/utils/node/reset-predicate-inputs.js +11 -0
- package/src/core/utils/node/unpause.js +11 -0
- package/src/core/utils/node/validate-spec.js +22 -0
- package/src/core/utils/options/resolve-field-keyed.js +115 -0
- package/src/core/utils/quantize/index.js +75 -0
- package/src/core/utils/stats/formulas.js +67 -0
- package/src/core/utils/stats/index.js +3 -0
- package/src/core/utils/validate/composer-validators.js +141 -0
- package/src/core/utils/validate/helpers.js +206 -0
- package/src/core/utils/validate/index.js +12 -0
- package/src/core/utils/validate/type-validators.js +538 -0
- package/src/core/utils/validate/validate.js +107 -0
- package/src/core/utils/validate/validators.js +177 -0
- package/src/core/wiring/assert-handle.js +57 -0
- package/src/core/wiring/assert-module.js +63 -0
- package/src/core/wiring/index.js +5 -0
- package/src/core/wiring/node-type-to-module.js +17 -0
- package/src/core/wiring/wire-emitters.js +256 -0
- package/src/core/wiring/wire-linear-graph.js +25 -0
- package/src/core/wiring/wire-node.js +157 -0
- package/src/core/wiring/wire-semantics.js +173 -0
- package/src/core/wiring/wire-storages.js +344 -0
- package/src/flow/check-output-collisions.js +93 -0
- package/src/flow/consts.js +9 -0
- package/src/flow/deep-clone-spec.js +64 -0
- package/src/flow/driver.js +171 -0
- package/src/flow/expand-single-input-array.js +76 -0
- package/src/flow/expand-template-spec.js +93 -0
- package/src/flow/flow-api-schemas.js +300 -0
- package/src/flow/flow.js +658 -0
- package/src/flow/for-each.js +225 -0
- package/src/flow/get-signature-pattern.js +31 -0
- package/src/flow/get-target-array.js +46 -0
- package/src/flow/inspect.js +67 -0
- package/src/flow/load-node-module.js +81 -0
- package/src/flow/make-node-method.js +112 -0
- package/src/flow/resolve-pick-by-field.js +77 -0
- package/src/flow/resolve-tunables.js +76 -0
- package/src/flow/run.js +392 -0
- package/src/flow/serialize.js +161 -0
- package/src/flow/signatures-schema.js +222 -0
- package/src/flow/spec-builder.js +65 -0
- package/src/flow/validate-triggers.js +250 -0
- package/src/flow/validate.js +261 -0
- package/src/nodes/accumulate/index.js +21 -0
- package/src/nodes/accumulate/init.js +48 -0
- package/src/nodes/accumulate/introspect.js +123 -0
- package/src/nodes/accumulate/publish-to.js +34 -0
- package/src/nodes/accumulate/recompute.js +27 -0
- package/src/nodes/accumulate/reset.js +25 -0
- package/src/nodes/accumulate/update.js +42 -0
- package/src/nodes/appraise/calibrate.js +64 -0
- package/src/nodes/appraise/decision.js +66 -0
- package/src/nodes/appraise/deviation.js +140 -0
- package/src/nodes/appraise/index.js +23 -0
- package/src/nodes/appraise/init.js +165 -0
- package/src/nodes/appraise/integrate.js +45 -0
- package/src/nodes/appraise/introspect.js +352 -0
- package/src/nodes/appraise/publish-to.js +93 -0
- package/src/nodes/appraise/receptor.js +69 -0
- package/src/nodes/appraise/recompute.js +79 -0
- package/src/nodes/appraise/reset.js +53 -0
- package/src/nodes/appraise/update.js +130 -0
- package/src/nodes/archive/csv/index.js +6 -0
- package/src/nodes/archive/csv/init.js +36 -0
- package/src/nodes/archive/csv/load.js +70 -0
- package/src/nodes/archive/csv/reset.js +14 -0
- package/src/nodes/archive/csv/simple-parser.js +216 -0
- package/src/nodes/archive/csv/test.js +173 -0
- package/src/nodes/archive/csv/update.js +63 -0
- package/src/nodes/butterworth-filter/index.js +18 -0
- package/src/nodes/butterworth-filter/init.js +184 -0
- package/src/nodes/butterworth-filter/introspect.js +228 -0
- package/src/nodes/butterworth-filter/publish-to.js +21 -0
- package/src/nodes/butterworth-filter/recompute.js +13 -0
- package/src/nodes/butterworth-filter/reset.js +30 -0
- package/src/nodes/butterworth-filter/update.js +38 -0
- package/src/nodes/categorize/index.js +19 -0
- package/src/nodes/categorize/init.js +58 -0
- package/src/nodes/categorize/introspect.js +177 -0
- package/src/nodes/categorize/publish-to.js +29 -0
- package/src/nodes/categorize/recompute.js +14 -0
- package/src/nodes/categorize/reset.js +16 -0
- package/src/nodes/categorize/update.js +53 -0
- package/src/nodes/controller/index.js +9 -0
- package/src/nodes/controller/init.js +55 -0
- package/src/nodes/controller/introspect.js +91 -0
- package/src/nodes/controller/publish-to.js +8 -0
- package/src/nodes/controller/recompute.js +9 -0
- package/src/nodes/controller/reset.js +17 -0
- package/src/nodes/controller/update.js +63 -0
- package/src/nodes/diff/index.js +19 -0
- package/src/nodes/diff/init.js +37 -0
- package/src/nodes/diff/introspect.js +132 -0
- package/src/nodes/diff/publish-to.js +21 -0
- package/src/nodes/diff/recompute.js +12 -0
- package/src/nodes/diff/reset.js +12 -0
- package/src/nodes/diff/update.js +37 -0
- package/src/nodes/digest-moments/formulas.js +12 -0
- package/src/nodes/digest-moments/index.js +14 -0
- package/src/nodes/digest-moments/init.js +67 -0
- package/src/nodes/digest-moments/introspect.js +154 -0
- package/src/nodes/digest-moments/publish-to.js +34 -0
- package/src/nodes/digest-moments/recompute.js +14 -0
- package/src/nodes/digest-moments/reset.js +14 -0
- package/src/nodes/digest-moments/update.js +120 -0
- package/src/nodes/dwell-time-tracker/index.js +14 -0
- package/src/nodes/dwell-time-tracker/init.js +59 -0
- package/src/nodes/dwell-time-tracker/introspect.js +137 -0
- package/src/nodes/dwell-time-tracker/publish-to.js +22 -0
- package/src/nodes/dwell-time-tracker/recompute.js +8 -0
- package/src/nodes/dwell-time-tracker/reset.js +23 -0
- package/src/nodes/dwell-time-tracker/update.js +103 -0
- package/src/nodes/emit-if/helpers.js +94 -0
- package/src/nodes/emit-if/index.js +16 -0
- package/src/nodes/emit-if/init.js +67 -0
- package/src/nodes/emit-if/introspect.js +102 -0
- package/src/nodes/emit-if/publish-to.js +13 -0
- package/src/nodes/emit-if/recompute.js +12 -0
- package/src/nodes/emit-if/reset.js +27 -0
- package/src/nodes/emit-if/update.js +93 -0
- package/src/nodes/es-correlation/index.js +19 -0
- package/src/nodes/es-correlation/init.js +73 -0
- package/src/nodes/es-correlation/introspect.js +176 -0
- package/src/nodes/es-correlation/publish-to.js +32 -0
- package/src/nodes/es-correlation/recompute.js +56 -0
- package/src/nodes/es-correlation/reset.js +27 -0
- package/src/nodes/es-correlation/update.js +110 -0
- package/src/nodes/es-mean/index.js +20 -0
- package/src/nodes/es-mean/init.js +58 -0
- package/src/nodes/es-mean/introspect.js +129 -0
- package/src/nodes/es-mean/publish-to.js +19 -0
- package/src/nodes/es-mean/recompute.js +16 -0
- package/src/nodes/es-mean/reset.js +24 -0
- package/src/nodes/es-mean/update.js +69 -0
- package/src/nodes/es-pairwise-correlation/index.js +19 -0
- package/src/nodes/es-pairwise-correlation/init.js +98 -0
- package/src/nodes/es-pairwise-correlation/introspect.js +187 -0
- package/src/nodes/es-pairwise-correlation/publish-to.js +31 -0
- package/src/nodes/es-pairwise-correlation/recompute.js +69 -0
- package/src/nodes/es-pairwise-correlation/reset.js +29 -0
- package/src/nodes/es-pairwise-correlation/update.js +89 -0
- package/src/nodes/es-stats/index.js +21 -0
- package/src/nodes/es-stats/init.js +105 -0
- package/src/nodes/es-stats/introspect.js +157 -0
- package/src/nodes/es-stats/publish-to.js +42 -0
- package/src/nodes/es-stats/recompute.js +74 -0
- package/src/nodes/es-stats/reset.js +49 -0
- package/src/nodes/es-stats/update-envelope.js +44 -0
- package/src/nodes/es-stats/update-welford.js +64 -0
- package/src/nodes/es-stats/update.js +59 -0
- package/src/nodes/index.js +83 -0
- package/src/nodes/invert-flag/index.js +20 -0
- package/src/nodes/invert-flag/init.js +36 -0
- package/src/nodes/invert-flag/introspect.js +115 -0
- package/src/nodes/invert-flag/publish-to.js +23 -0
- package/src/nodes/invert-flag/recompute.js +12 -0
- package/src/nodes/invert-flag/reset.js +12 -0
- package/src/nodes/invert-flag/update.js +32 -0
- package/src/nodes/kalman1d/index.js +20 -0
- package/src/nodes/kalman1d/init.js +103 -0
- package/src/nodes/kalman1d/introspect.js +250 -0
- package/src/nodes/kalman1d/publish-to.js +34 -0
- package/src/nodes/kalman1d/recompute.js +23 -0
- package/src/nodes/kalman1d/reset.js +29 -0
- package/src/nodes/kalman1d/update.js +144 -0
- package/src/nodes/kernel/index.js +18 -0
- package/src/nodes/kernel/init.js +66 -0
- package/src/nodes/kernel/introspect.js +163 -0
- package/src/nodes/kernel/presets.js +74 -0
- package/src/nodes/kernel/publish-to.js +25 -0
- package/src/nodes/kernel/recompute.js +14 -0
- package/src/nodes/kernel/reset.js +20 -0
- package/src/nodes/kernel/update.js +69 -0
- package/src/nodes/lag/index.js +21 -0
- package/src/nodes/lag/init.js +87 -0
- package/src/nodes/lag/introspect.js +202 -0
- package/src/nodes/lag/publish-to.js +64 -0
- package/src/nodes/lag/recompute.js +33 -0
- package/src/nodes/lag/reset.js +49 -0
- package/src/nodes/lag/update.js +133 -0
- package/src/nodes/median3/index.js +19 -0
- package/src/nodes/median3/init.js +40 -0
- package/src/nodes/median3/introspect.js +116 -0
- package/src/nodes/median3/publish-to.js +21 -0
- package/src/nodes/median3/recompute.js +12 -0
- package/src/nodes/median3/reset.js +15 -0
- package/src/nodes/median3/update.js +58 -0
- package/src/nodes/moments-digest/copy-from.js +12 -0
- package/src/nodes/moments-digest/flush.js +17 -0
- package/src/nodes/moments-digest/index.js +132 -0
- package/src/nodes/moments-digest/init.js +84 -0
- package/src/nodes/moments-digest/introspect.js +122 -0
- package/src/nodes/moments-digest/prelude.js +28 -0
- package/src/nodes/moments-digest/publish-to.js +31 -0
- package/src/nodes/moments-digest/recompute.js +9 -0
- package/src/nodes/moments-digest/reset.js +20 -0
- package/src/nodes/moments-digest/update.js +182 -0
- package/src/nodes/node-names.js +67 -0
- package/src/nodes/page-hinkley/index.js +20 -0
- package/src/nodes/page-hinkley/init.js +66 -0
- package/src/nodes/page-hinkley/introspect.js +193 -0
- package/src/nodes/page-hinkley/publish-to.js +28 -0
- package/src/nodes/page-hinkley/recompute.js +14 -0
- package/src/nodes/page-hinkley/reset.js +23 -0
- package/src/nodes/page-hinkley/update.js +105 -0
- package/src/nodes/pass-if/index.js +21 -0
- package/src/nodes/pass-if/init.js +37 -0
- package/src/nodes/pass-if/introspect.js +93 -0
- package/src/nodes/pass-if/publish-to.js +15 -0
- package/src/nodes/pass-if/recompute.js +15 -0
- package/src/nodes/pass-if/reset.js +18 -0
- package/src/nodes/pass-if/update.js +37 -0
- package/src/nodes/persist-if/helpers.js +111 -0
- package/src/nodes/persist-if/index.js +9 -0
- package/src/nodes/persist-if/init.js +98 -0
- package/src/nodes/persist-if/introspect.js +106 -0
- package/src/nodes/persist-if/publish-to.js +21 -0
- package/src/nodes/persist-if/recompute.js +20 -0
- package/src/nodes/persist-if/reset.js +35 -0
- package/src/nodes/persist-if/update.js +112 -0
- package/src/nodes/persistence-check/index.js +14 -0
- package/src/nodes/persistence-check/init.js +38 -0
- package/src/nodes/persistence-check/introspect.js +144 -0
- package/src/nodes/persistence-check/publish-to.js +11 -0
- package/src/nodes/persistence-check/recompute.js +7 -0
- package/src/nodes/persistence-check/reset.js +12 -0
- package/src/nodes/persistence-check/update.js +57 -0
- package/src/nodes/process-index/index.js +15 -0
- package/src/nodes/process-index/init.js +48 -0
- package/src/nodes/process-index/introspect.js +182 -0
- package/src/nodes/process-index/publish-to.js +29 -0
- package/src/nodes/process-index/recompute.js +8 -0
- package/src/nodes/process-index/reset.js +8 -0
- package/src/nodes/process-index/update.js +75 -0
- package/src/nodes/ratio/index.js +14 -0
- package/src/nodes/ratio/init.js +37 -0
- package/src/nodes/ratio/introspect.js +155 -0
- package/src/nodes/ratio/publish-to.js +15 -0
- package/src/nodes/ratio/recompute.js +7 -0
- package/src/nodes/ratio/reset.js +7 -0
- package/src/nodes/ratio/update.js +47 -0
- package/src/nodes/sanitize/index.js +82 -0
- package/src/nodes/sanitize/init.js +82 -0
- package/src/nodes/sanitize/introspect.js +194 -0
- package/src/nodes/sanitize/publish-to.js +15 -0
- package/src/nodes/sanitize/recompute.js +8 -0
- package/src/nodes/sanitize/reset.js +14 -0
- package/src/nodes/sanitize/update.js +102 -0
- package/src/nodes/spike-guard/index.js +14 -0
- package/src/nodes/spike-guard/init.js +47 -0
- package/src/nodes/spike-guard/introspect.js +130 -0
- package/src/nodes/spike-guard/publish-to.js +35 -0
- package/src/nodes/spike-guard/recompute.js +14 -0
- package/src/nodes/spike-guard/reset.js +19 -0
- package/src/nodes/spike-guard/update.js +89 -0
- package/src/nodes/sse-emitter/index.js +7 -0
- package/src/nodes/sse-emitter/init.js +105 -0
- package/src/nodes/sse-emitter/introspect.js +134 -0
- package/src/nodes/sse-emitter/publish-to.js +6 -0
- package/src/nodes/sse-emitter/recompute.js +16 -0
- package/src/nodes/sse-emitter/reset.js +19 -0
- package/src/nodes/sse-emitter/update.js +69 -0
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +36 -0
- package/src/nodes/sse-emitter/utils/build-message.js +31 -0
- package/src/nodes/sse-emitter/utils/connection-manager.js +45 -0
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +20 -0
- package/src/nodes/sse-emitter/utils/health-monitor.js +38 -0
- package/src/nodes/sse-emitter/utils/qos-handler.js +41 -0
- package/src/nodes/state-change-detector/helpers.js +35 -0
- package/src/nodes/state-change-detector/index.js +12 -0
- package/src/nodes/state-change-detector/init.js +56 -0
- package/src/nodes/state-change-detector/introspect.js +146 -0
- package/src/nodes/state-change-detector/publish-to.js +20 -0
- package/src/nodes/state-change-detector/recompute.js +6 -0
- package/src/nodes/state-change-detector/reset.js +20 -0
- package/src/nodes/state-change-detector/update.js +97 -0
- package/src/nodes/sw-stats/index.js +14 -0
- package/src/nodes/sw-stats/init.js +51 -0
- package/src/nodes/sw-stats/introspect.js +131 -0
- package/src/nodes/sw-stats/publish-to.js +45 -0
- package/src/nodes/sw-stats/recompute.js +33 -0
- package/src/nodes/sw-stats/reset.js +15 -0
- package/src/nodes/sw-stats/update.js +39 -0
- package/src/nodes/swing-watch/index.js +17 -0
- package/src/nodes/swing-watch/init.js +144 -0
- package/src/nodes/swing-watch/introspect.js +188 -0
- package/src/nodes/swing-watch/publish-to.js +53 -0
- package/src/nodes/swing-watch/recompute.js +22 -0
- package/src/nodes/swing-watch/reset.js +55 -0
- package/src/nodes/swing-watch/update.js +767 -0
- package/src/nodes/tally/index.js +15 -0
- package/src/nodes/tally/init.js +32 -0
- package/src/nodes/tally/introspect.js +116 -0
- package/src/nodes/tally/publish-to.js +28 -0
- package/src/nodes/tally/recompute.js +10 -0
- package/src/nodes/tally/reset.js +10 -0
- package/src/nodes/tally/update.js +44 -0
- package/src/nodes/threshold/index.js +14 -0
- package/src/nodes/threshold/init.js +121 -0
- package/src/nodes/threshold/introspect.js +192 -0
- package/src/nodes/threshold/publish-to.js +11 -0
- package/src/nodes/threshold/recompute.js +8 -0
- package/src/nodes/threshold/reset.js +14 -0
- package/src/nodes/threshold/update.js +54 -0
- package/src/nodes/transform/helpers.js +39 -0
- package/src/nodes/transform/index.js +15 -0
- package/src/nodes/transform/init.js +35 -0
- package/src/nodes/transform/introspect.js +118 -0
- package/src/nodes/transform/publish-to.js +23 -0
- package/src/nodes/transform/recompute.js +12 -0
- package/src/nodes/transform/reset.js +11 -0
- package/src/nodes/transform/update.js +32 -0
- package/src/nodes/trend/compute-confidence.js +112 -0
- package/src/nodes/trend/index.js +14 -0
- package/src/nodes/trend/init.js +81 -0
- package/src/nodes/trend/introspect.js +151 -0
- package/src/nodes/trend/publish-to.js +22 -0
- package/src/nodes/trend/recompute.js +15 -0
- package/src/nodes/trend/reset.js +26 -0
- package/src/nodes/trend/update.js +93 -0
- package/src/nodes/tw-stats/copy-from.js +16 -0
- package/src/nodes/tw-stats/flush.js +12 -0
- package/src/nodes/tw-stats/index.js +41 -0
- package/src/nodes/tw-stats/init.js +92 -0
- package/src/nodes/tw-stats/introspect.js +171 -0
- package/src/nodes/tw-stats/publish-to.js +108 -0
- package/src/nodes/tw-stats/recompute.js +9 -0
- package/src/nodes/tw-stats/reset.js +16 -0
- package/src/nodes/tw-stats/update.js +113 -0
- package/src/nodes/unbalance/index.js +15 -0
- package/src/nodes/unbalance/init.js +59 -0
- package/src/nodes/unbalance/introspect.js +167 -0
- package/src/nodes/unbalance/publish-to.js +52 -0
- package/src/nodes/unbalance/recompute.js +11 -0
- package/src/nodes/unbalance/reset.js +10 -0
- package/src/nodes/unbalance/update.js +83 -0
- package/src/nodes/vector-distance/index.js +14 -0
- package/src/nodes/vector-distance/init.js +54 -0
- package/src/nodes/vector-distance/introspect.js +149 -0
- package/src/nodes/vector-distance/publish-to.js +35 -0
- package/src/nodes/vector-distance/recompute.js +11 -0
- package/src/nodes/vector-distance/reset.js +20 -0
- package/src/nodes/vector-distance/update.js +139 -0
- package/src/nodes/winnow/index.js +12 -0
- package/src/nodes/winnow/init.js +82 -0
- package/src/nodes/winnow/introspect.js +253 -0
- package/src/nodes/winnow/publish-to.js +51 -0
- package/src/nodes/winnow/recompute.js +14 -0
- package/src/nodes/winnow/reset.js +34 -0
- package/src/nodes/winnow/update.js +203 -0
- package/src/tools/training/features.js +227 -0
- package/src/tools/training/index.js +21 -0
- package/src/tools/training/logistic.js +238 -0
- package/src/tools/training/metrics.js +126 -0
- package/src/tools/training/scale.js +239 -0
- package/src/tools/training/validate.js +103 -0
- package/src/windowing/count-sliding/create.js +16 -0
- package/src/windowing/count-sliding/get-reverse-chronological-index.js +19 -0
- package/src/windowing/count-sliding/index.js +6 -0
- package/src/windowing/count-sliding/is-not-full.js +5 -0
- package/src/windowing/count-sliding/push.js +15 -0
- package/src/windowing/count-sliding/reset.js +11 -0
- package/src/windowing/count-sliding/size.js +5 -0
- package/src/windowing/time-sliding/create.js +25 -0
- package/src/windowing/time-sliding/push.js +42 -0
- package/CODE_OF_CONDUCT.md +0 -46
- package/CONTRIBUTING.md +0 -132
- package/docs/.nojekyll +0 -0
- package/docs/CONTRIBUTING.md +0 -134
- package/docs/README.md +0 -132
- package/docs/_coverpage.md +0 -26
- package/docs/_media/bg.png +0 -0
- package/docs/_media/wink-logo.png +0 -0
- package/docs/_navbar.md +0 -14
- package/docs/_sidebar.md +0 -2
- package/docs/index.html +0 -44
- package/eslint.config.js +0 -283
|
@@ -0,0 +1,1251 @@
|
|
|
1
|
+
// core/storage-manager/questdb/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview QuestDB storage adapter for time-series data persistence.
|
|
5
|
+
*
|
|
6
|
+
* Creates storage interface that integrates with partition-manager:
|
|
7
|
+
* - ensureTables: Creates tables via PostgreSQL wire protocol at startup
|
|
8
|
+
* - write(insightType, msg, partitionId): Writes row via ILP, sync return
|
|
9
|
+
* per the ADR-018 sink contract — `{ ok: true }` on success,
|
|
10
|
+
* `{ ok: false, error: { code, message } }` on failure.
|
|
11
|
+
* - flush(): Manual flush (in manual mode) or no-op (in auto mode)
|
|
12
|
+
* - shutdown(): Graceful shutdown
|
|
13
|
+
* - getPressure(): Buffer fill ratio in [0, 1] per ADR-018 (sync, O(1),
|
|
14
|
+
* allocation-free).
|
|
15
|
+
*
|
|
16
|
+
* Two flush modes:
|
|
17
|
+
* - auto: QuestDB client handles flushing based on rows/interval
|
|
18
|
+
* - manual: Application controls flush timing via idle timer
|
|
19
|
+
*
|
|
20
|
+
* Adapter contract (ADR-018, structured-sink role):
|
|
21
|
+
* - Module-level exports per the ADR-018 module surface: `id`, `configSchema`,
|
|
22
|
+
* `createStorage`, `durabilityClass`, `semanticsRequirement`, and the
|
|
23
|
+
* default aggregate (`questdbAdapter`) referencing the same constants.
|
|
24
|
+
* - Durability class (ADR-018): `'in-memory'` — rows accepted by `write()`
|
|
25
|
+
* live in the ILP client's in-process buffer until a flush reaches the
|
|
26
|
+
* server.
|
|
27
|
+
* A process crash loses every un-flushed row; the shutdown drain and
|
|
28
|
+
* its classified loss reporting exist precisely to shrink that window.
|
|
29
|
+
* - Hot-path `write` returns sync per ADR-013; the partition-manager never
|
|
30
|
+
* awaits a Promise on the per-message path.
|
|
31
|
+
* - `getPressure()` returns `(bufferedRows + inFlightRows) / autoFlushRows`,
|
|
32
|
+
* clamped to 1.0. `bufferedRows` counts rows awaiting the next flush and
|
|
33
|
+
* increments synchronously when the persist plan opens a row; a flush call
|
|
34
|
+
* moves its rows to `inFlightRows` (the client copies them out of the
|
|
35
|
+
* buffer synchronously — copy-out), where they stay until the flush
|
|
36
|
+
* settles. In auto mode the counter mirrors QuestDB's internal flush
|
|
37
|
+
* trigger via a heuristic reset at the `autoFlushRows` boundary; the
|
|
38
|
+
* `auto_flush_interval` trigger is not directly observable but self-heals
|
|
39
|
+
* via `checkIdleFlush` after `idleFlushAfterMs` of write-idle.
|
|
40
|
+
*
|
|
41
|
+
* Health-status semantics (uniform across sinks):
|
|
42
|
+
* - `getHealth()` returns `{status, connected, pressure, ...}` — the ADR-018
|
|
43
|
+
* health floor.
|
|
44
|
+
* - `connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD`.
|
|
45
|
+
* QuestDB has no persistent connection state to inspect (ILP sender is
|
|
46
|
+
* fire-and-forget); `connected` is *derived* from recent write success.
|
|
47
|
+
* - Status derivation:
|
|
48
|
+
* `red` if `!connected` (transport unhealthy or shutting down)
|
|
49
|
+
* `yellow` if `pressure >= 0.66` (threshold shared with ADR-020's
|
|
50
|
+
* Draft yield proposal) OR
|
|
51
|
+
* `consecutiveWriteErrors >= 1` (any error elevates concern)
|
|
52
|
+
* `green` otherwise
|
|
53
|
+
* - The counter resets to 0 on every successful write, so a single recovery
|
|
54
|
+
* write returns `status` to `green`.
|
|
55
|
+
*
|
|
56
|
+
* `err.code` vocabulary (per-adapter; ADR-018 has each adapter document
|
|
57
|
+
* its own codes in this header):
|
|
58
|
+
*
|
|
59
|
+
* Setup-time throws (ADR-018 fail-fast setup):
|
|
60
|
+
* - `INVALID_CONFIG` — the supplied configuration does not
|
|
61
|
+
* work. Five current sub-cases:
|
|
62
|
+
* (a) required transport URL missing (`ilpUrl`, `pgUrl`);
|
|
63
|
+
* (b) PostgreSQL endpoint answered but rejected the connection
|
|
64
|
+
* — wrong credentials or a protocol-level refusal;
|
|
65
|
+
* (c) a column declares an unsupported `type`, or a `float64`
|
|
66
|
+
* column declares a non-positive `resolution` (see
|
|
67
|
+
* `assert-columns.js`);
|
|
68
|
+
* (d) a table or column name the client's own ILP rules reject.
|
|
69
|
+
* Checked at plan build by driving each name through a
|
|
70
|
+
* throwaway client buffer (`assertIlpNames` in
|
|
71
|
+
* `persist-plan.js`) — a bad name fails deployment instead
|
|
72
|
+
* of wedging the sender mid-row at runtime;
|
|
73
|
+
* (e) an insightType uses the reserved column name `assetId`.
|
|
74
|
+
* Composer writes that column itself, from the partition id.
|
|
75
|
+
* Checked at plan build in `persist-plan.js`.
|
|
76
|
+
* Operator remediation: fix the supplied config or the relevant
|
|
77
|
+
* `QUESTDB_*` env var. The underlying transport error (sub-case b)
|
|
78
|
+
* is preserved on `err.cause` for diagnostics.
|
|
79
|
+
* - `TRANSPORT_UNREACHABLE` — the PostgreSQL endpoint did not answer
|
|
80
|
+
* at setup: nothing listening on the port, host unresolvable, no
|
|
81
|
+
* route, or the attempt timed out (a Node syscall code —
|
|
82
|
+
* `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, ...). Distinct from
|
|
83
|
+
* `INVALID_CONFIG` per the one split the ADR-018 error vocabulary
|
|
84
|
+
* mandates: the
|
|
85
|
+
* connection string may be fine — check the network, the firewall,
|
|
86
|
+
* whether QuestDB is running. Underlying error on `err.cause`.
|
|
87
|
+
* - `MISSING_ASSET_CLASS` — `assetClass` not provided to `createStorage`.
|
|
88
|
+
* Distinct from `INVALID_CONFIG` because operator remediation differs:
|
|
89
|
+
* they need to add `.assetClass(assetClassDef)` to the flow, not edit env
|
|
90
|
+
* vars or storage config.
|
|
91
|
+
* - `SCHEMA_ERROR` — DDL `CREATE TABLE` failed for a reason other
|
|
92
|
+
* than already-exists (see `ensure-tables.js`).
|
|
93
|
+
*
|
|
94
|
+
* Runtime returns:
|
|
95
|
+
* - `INVALID_INSIGHT_TYPE` — sync; the insightType has no persist plan
|
|
96
|
+
* (typically a config typo or asset-class drift).
|
|
97
|
+
* - `SEND_FAILED` — sync; the persist plan threw while invoking
|
|
98
|
+
* sender methods (ILP buffer overflow, type coercion failure, etc.).
|
|
99
|
+
* - `INVALID_TIMESTAMP` — reserved; today persist-plan handles invalid
|
|
100
|
+
* timestamps via `onWarning + skip-row`. It may later be surfaced as a
|
|
101
|
+
* hard return error.
|
|
102
|
+
*
|
|
103
|
+
* Shutdown-time throws (ADR-018 — a clean shutdown resolve means
|
|
104
|
+
* everything buffered or in flight was delivered; both carry
|
|
105
|
+
* `dropped: { count }`, exact). Shutdown settles every unsettled flush
|
|
106
|
+
* (idle, recovery) plus one final flush, raced against `{ timeout }`;
|
|
107
|
+
* its outcome is latched so repeated calls cannot contradict it:
|
|
108
|
+
* - `DELIVERY_FAILED` — a flush failed; first flush error on `cause`.
|
|
109
|
+
* - `SHUTDOWN_TIMEOUT` — delivery did not settle within the caller's
|
|
110
|
+
* `{ timeout }` (a send against an unreachable server never settles —
|
|
111
|
+
* see the mid-row recovery section below).
|
|
112
|
+
*
|
|
113
|
+
* Column-internal facts consumed (the ADR-018 column-internal facts
|
|
114
|
+
* pattern):
|
|
115
|
+
*
|
|
116
|
+
* QuestDB's top-level dependency on the asset class is declared in the
|
|
117
|
+
* `semanticsRequirement` export below: `name`, `columns`, `insightTypes`.
|
|
118
|
+
* Inside `columns`, QuestDB additionally reads two fields per column:
|
|
119
|
+
*
|
|
120
|
+
* - `columns.*.type` — read by `ensure-tables.js` (DDL type mapping
|
|
121
|
+
* `float64 → DOUBLE`, `int64 → LONG`, etc.) and by `persist-plan.js`
|
|
122
|
+
* (writer dispatch + numeric/non-numeric branch). Asserted at startup
|
|
123
|
+
* by `assert-columns.js`: must be present and a key in `DDL_TYPES`;
|
|
124
|
+
* throws `INVALID_CONFIG` otherwise.
|
|
125
|
+
* - `columns.*.resolution` — read by `persist-plan.js` (passed to the
|
|
126
|
+
* resolution-aware float64 writer factory in `writers.js`). Asserted
|
|
127
|
+
* at startup by `assert-columns.js`: when present on a `float64`
|
|
128
|
+
* column, must be a positive finite number; absent is fine
|
|
129
|
+
* (passthrough). Throws `INVALID_CONFIG` otherwise.
|
|
130
|
+
*
|
|
131
|
+
* Future column-internal reads (e.g., `unit`, `physicalRange`) would
|
|
132
|
+
* extend this section and add corresponding assertions in
|
|
133
|
+
* `assert-columns.js`.
|
|
134
|
+
*
|
|
135
|
+
* Async-rejection containment:
|
|
136
|
+
* - QuestDB's `sender.at(...)` is `async` — the buffer mutation is sync but
|
|
137
|
+
* the trailing `await this.tryFlush()` can fire a network flush. To prevent
|
|
138
|
+
* silent loss when that flush fails, `persist-plan.js` attaches a `.catch()`
|
|
139
|
+
* to the `at()` Promise and routes the failure to `onDeliveryFailure` when
|
|
140
|
+
* provided — otherwise it throws a classified `DELIVERY_FAILED` so the loss
|
|
141
|
+
* is loud. See `persist-plan.js` for the wrapper.
|
|
142
|
+
*
|
|
143
|
+
* Mid-row recovery (ADR-018 — a rejected message costs that message,
|
|
144
|
+
* nothing else):
|
|
145
|
+
* - A persist plan that throws between sender.table() and sender.at() leaves
|
|
146
|
+
* the client holding a half-written row. Without recovery, every later
|
|
147
|
+
* write fails with "Table name has already been set" — the 2026-06-10
|
|
148
|
+
* silent-write-failure incident lost 98.6% of a replay's rows this way.
|
|
149
|
+
* - write()'s catch therefore calls recoverSender(), which composes two
|
|
150
|
+
* documented client calls: flush() ships every COMPLETED row out of the
|
|
151
|
+
* buffer (the copy-out happens synchronously, before any network I/O; the
|
|
152
|
+
* client documents that an unfinished row stays behind), then reset()
|
|
153
|
+
* clears the buffer — at that point holding only the broken stub — and
|
|
154
|
+
* lowers the client's row-in-progress flags.
|
|
155
|
+
* - The client (4.2.0) has no row-cancel API, while its sibling clients do
|
|
156
|
+
* (.NET CancelRow, Rust/C rewind_to_marker, Java recovers automatically).
|
|
157
|
+
* Upstream issue #60 tracks the gap:
|
|
158
|
+
* https://github.com/questdb/nodejs-questdb-client/issues/60
|
|
159
|
+
* When a release ships cancelRow(), recoverSender() becomes that one call.
|
|
160
|
+
*
|
|
161
|
+
* Integration with persist-if node:
|
|
162
|
+
* - persist-if calls: state.storage.write(insightType, msg, partitionId)
|
|
163
|
+
* - This maps to: persistPlans[insightType](sender, msg, partitionId)
|
|
164
|
+
* - Reads `result.ok`, `result.error.code`, `result.error.message` per ADR-018.
|
|
165
|
+
*
|
|
166
|
+
* Storage Adapter Interface:
|
|
167
|
+
* - Exports `questdbAdapter` with `id` and `createStorage` for flow DSL
|
|
168
|
+
* - `createStorage(config)` is async, handles table creation and ILP connection
|
|
169
|
+
*
|
|
170
|
+
* Reading QuestDB rows back via PostgreSQL — the timezone gotcha:
|
|
171
|
+
*
|
|
172
|
+
* QuestDB stores TIMESTAMP columns as microseconds since the Unix epoch
|
|
173
|
+
* (no timezone). When you read those columns over the PostgreSQL wire
|
|
174
|
+
* protocol, QuestDB reports them as `TIMESTAMP without time zone`. The
|
|
175
|
+
* `pg` Node.js library treats that type as local-time and silently
|
|
176
|
+
* shifts the value by the local UTC offset when constructing a Date —
|
|
177
|
+
* so a non-UTC client sees a wrong `Date.getTime()`.
|
|
178
|
+
*
|
|
179
|
+
* The simplest fix: cast the column in QuestDB itself so pg never gets
|
|
180
|
+
* a chance to interpret it. `SELECT timestamp::long FROM ...` returns
|
|
181
|
+
* bigint microseconds, untouched. Divide by 1000 to recover ms.
|
|
182
|
+
*
|
|
183
|
+
* Example (Node.js with `pg`):
|
|
184
|
+
* SELECT _harnessId, timestamp::long AS ts_us, value FROM samples
|
|
185
|
+
* Then in JS: `tsMs = parseInt(row.ts_us, 10) / 1000;`
|
|
186
|
+
*
|
|
187
|
+
* Tests in non-UTC timezones almost always need this cast. See
|
|
188
|
+
* `src/core/source-manager/test-harness/test/e2e-contract-harness.specs.js`
|
|
189
|
+
* for a working example.
|
|
190
|
+
*
|
|
191
|
+
* @see https://questdb.com/docs/clients/ingest-node/
|
|
192
|
+
* @see docs/architecture/storage-layer.md
|
|
193
|
+
* @see ADR-018
|
|
194
|
+
*/
|
|
195
|
+
|
|
196
|
+
import { Sender } from '@questdb/nodejs-client';
|
|
197
|
+
import pg from 'pg';
|
|
198
|
+
|
|
199
|
+
import { ENV_VARS } from '../../env-vars.js';
|
|
200
|
+
import { validators } from '../../utils/validate/index.js';
|
|
201
|
+
import { buildPersistPlans } from './persist-plan.js';
|
|
202
|
+
import { ensureTables } from './ensure-tables.js';
|
|
203
|
+
import { assertColumnFacts } from './assert-columns.js';
|
|
204
|
+
|
|
205
|
+
// ============================================================================
|
|
206
|
+
// HOT-PATH SINGLETONS
|
|
207
|
+
// ============================================================================
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Singleton success result reused on every successful write. Hot-path zero
|
|
211
|
+
* allocation per ADR-013 / ADR-004. Plain literal — not frozen (V8 hot paths
|
|
212
|
+
* handle plain objects more predictably; no caller mutates this).
|
|
213
|
+
* @type {{ok: true}}
|
|
214
|
+
*/
|
|
215
|
+
const RESULT_OK = { ok: true };
|
|
216
|
+
|
|
217
|
+
// Error results (INVALID_INSIGHT_TYPE, SEND_FAILED) are constructed per-call
|
|
218
|
+
// because each carries dynamic content (the offending insightType name and the
|
|
219
|
+
// underlying err.message respectively). These paths are rare; per-occurrence
|
|
220
|
+
// allocation on errors is acceptable. The RESULT_OK singleton above covers the
|
|
221
|
+
// per-message hot path.
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Health status thresholds. consecutiveWriteErrors crossing the YELLOW
|
|
225
|
+
* threshold elevates status to 'yellow'; crossing the RED threshold flips
|
|
226
|
+
* `connected` to false (and therefore `status` to 'red'). Tuned for "any
|
|
227
|
+
* error is worth flagging" + "sustained errors mean the transport is gone."
|
|
228
|
+
*/
|
|
229
|
+
const HEALTH_ERROR_YELLOW_THRESHOLD = 1;
|
|
230
|
+
const HEALTH_ERROR_RED_THRESHOLD = 5;
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Pressure threshold above which `status` elevates to at least 'yellow'.
|
|
234
|
+
* The value matches the pressure-aware-yield design (ADR-020, still a
|
|
235
|
+
* Draft). That design proposes 0.66 as the pressure level where the
|
|
236
|
+
* flow would start yielding to let sinks drain; the number still needs
|
|
237
|
+
* benchmark confirmation. Today the yield trigger is time-only (ADR-024), so
|
|
238
|
+
* this alignment is forward-looking, not a description of current yield
|
|
239
|
+
* behaviour. Numeric — the constant exists once here rather than scattered
|
|
240
|
+
* as a magic number.
|
|
241
|
+
*/
|
|
242
|
+
const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
|
|
243
|
+
|
|
244
|
+
// ============================================================================
|
|
245
|
+
// CONFIGURATION BUILDER
|
|
246
|
+
// ============================================================================
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Build QuestDB Sender configuration from options.
|
|
250
|
+
*
|
|
251
|
+
* @param {Object} options - Storage options
|
|
252
|
+
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
253
|
+
* @param {string} options.flushMode - 'auto' or 'manual'
|
|
254
|
+
* @param {number} [options.autoFlushRows] - Rows before auto-flush
|
|
255
|
+
* @param {number} [options.autoFlushIntervalMs] - Time interval for auto-flush
|
|
256
|
+
* @param {number} [options.maxBufSize] - Maximum buffer size
|
|
257
|
+
* @param {number} [options.retryTimeout] - Retry timeout in ms
|
|
258
|
+
* @returns {string} Configuration string for Sender.fromConfig()
|
|
259
|
+
*/
|
|
260
|
+
const buildSenderConfig = function ( options ) {
|
|
261
|
+
const { ilpUrl, flushMode, autoFlushRows, autoFlushIntervalMs, maxBufSize, retryTimeout } = options;
|
|
262
|
+
|
|
263
|
+
// Base configuration: HTTP transport
|
|
264
|
+
let config = `http::addr=${ilpUrl};`;
|
|
265
|
+
|
|
266
|
+
// Auto-flush configuration
|
|
267
|
+
if ( flushMode === 'auto' ) {
|
|
268
|
+
// Use QuestDB's built-in auto-flush
|
|
269
|
+
if ( autoFlushRows !== undefined ) {
|
|
270
|
+
config += `auto_flush_rows=${autoFlushRows};`;
|
|
271
|
+
}
|
|
272
|
+
if ( autoFlushIntervalMs !== undefined ) {
|
|
273
|
+
config += `auto_flush_interval=${autoFlushIntervalMs};`;
|
|
274
|
+
}
|
|
275
|
+
} else {
|
|
276
|
+
// Manual mode: disable auto-flush, we control timing
|
|
277
|
+
config += 'auto_flush=off;';
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// Optional buffer/retry settings
|
|
281
|
+
if ( maxBufSize !== undefined ) {
|
|
282
|
+
config += `init_buf_size=${maxBufSize};`;
|
|
283
|
+
}
|
|
284
|
+
if ( retryTimeout !== undefined ) {
|
|
285
|
+
config += `retry_timeout=${retryTimeout};`;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
return config;
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
// ============================================================================
|
|
292
|
+
// STORAGE FACTORY
|
|
293
|
+
// ============================================================================
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Race a final flush against the shutdown time budget. No budget
|
|
297
|
+
* (0/undefined) means no enforcement — the await is unbounded, preserving
|
|
298
|
+
* direct-caller behavior. On timeout the flush promise is deliberately
|
|
299
|
+
* left pending: the client's send may never settle (see the file header),
|
|
300
|
+
* and the process is shutting down anyway. Its eventual rejection, if
|
|
301
|
+
* any, is absorbed by the race's own handlers — never an
|
|
302
|
+
* unhandledRejection.
|
|
303
|
+
*
|
|
304
|
+
* @param {Promise} flushPromise - the in-flight sender.flush()
|
|
305
|
+
* @param {number} timeoutMs - budget in ms; 0/absent disables the race
|
|
306
|
+
* @returns {Promise} settles with the flush, or rejects SHUTDOWN_TIMEOUT
|
|
307
|
+
*/
|
|
308
|
+
const raceFlushTimeout = function ( flushPromise, timeoutMs ) {
|
|
309
|
+
if ( !( timeoutMs > 0 ) ) {
|
|
310
|
+
return flushPromise;
|
|
311
|
+
}
|
|
312
|
+
return new Promise( function ( resolve, reject ) {
|
|
313
|
+
const timer = setTimeout( function () {
|
|
314
|
+
const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
|
|
315
|
+
err.code = 'SHUTDOWN_TIMEOUT';
|
|
316
|
+
reject( err );
|
|
317
|
+
}, timeoutMs );
|
|
318
|
+
timer.unref();
|
|
319
|
+
flushPromise.then(
|
|
320
|
+
function ( value ) {
|
|
321
|
+
clearTimeout( timer );
|
|
322
|
+
resolve( value );
|
|
323
|
+
},
|
|
324
|
+
|
|
325
|
+
/* c8 ignore start -- unreachable from the sole caller:
|
|
326
|
+
doShutdown races a Promise.all over waits that absorb
|
|
327
|
+
their own rejections (that is what keeps the dropped
|
|
328
|
+
count exact), so the raced promise cannot reject today.
|
|
329
|
+
The handler stays because this utility's contract is
|
|
330
|
+
generic — without it, a rejecting promise from a future
|
|
331
|
+
caller would become an unhandled rejection and a race
|
|
332
|
+
that never settles. */
|
|
333
|
+
function ( err ) {
|
|
334
|
+
clearTimeout( timer );
|
|
335
|
+
reject( err );
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/* c8 ignore stop */
|
|
339
|
+
);
|
|
340
|
+
} );
|
|
341
|
+
}; // raceFlushTimeout()
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Node syscall codes that mean the PostgreSQL endpoint did not answer.
|
|
345
|
+
* Used at setup to classify a connect failure as TRANSPORT_UNREACHABLE
|
|
346
|
+
* (the one split the ADR-018 error vocabulary mandates — see the
|
|
347
|
+
* connect wrap below).
|
|
348
|
+
* Module-level Set: allocated once at load, membership check at setup.
|
|
349
|
+
*
|
|
350
|
+
* @type {Set<string>}
|
|
351
|
+
*/
|
|
352
|
+
const NETWORK_ERROR_CODES = new Set( [
|
|
353
|
+
'ECONNREFUSED',
|
|
354
|
+
'ENOTFOUND',
|
|
355
|
+
'ETIMEDOUT',
|
|
356
|
+
'EHOSTUNREACH',
|
|
357
|
+
'ENETUNREACH',
|
|
358
|
+
'ECONNRESET',
|
|
359
|
+
'EAI_AGAIN',
|
|
360
|
+
'EPIPE'
|
|
361
|
+
] );
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Create QuestDB storage adapter.
|
|
365
|
+
*
|
|
366
|
+
* @param {Object} assetClass - Asset class definition with columns and insightTypes
|
|
367
|
+
* @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
|
|
368
|
+
* @param {Object} options - Configuration options
|
|
369
|
+
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
370
|
+
* @param {string} options.pgUrl - PostgreSQL endpoint (host:port)
|
|
371
|
+
* @param {string} [options.flushMode='auto'] - 'auto' or 'manual'
|
|
372
|
+
* @param {number} [options.idleFlushAfterMs=5000] - Idle time before flush (manual mode)
|
|
373
|
+
* @param {number} [options.idleFlushCheckMs=1000] - Idle check interval (manual mode)
|
|
374
|
+
* @param {number} [options.autoFlushRows] - Rows before auto-flush (auto mode)
|
|
375
|
+
* @param {number} [options.autoFlushIntervalMs] - Time interval for auto-flush (auto mode)
|
|
376
|
+
* @param {number} [options.maxBufSize] - Maximum buffer size
|
|
377
|
+
* @param {number} [options.retryTimeout] - Retry timeout in ms
|
|
378
|
+
* @param {string} [options.partitionBy='DAY'] - Table partition interval
|
|
379
|
+
* @param {function} [options.onWarning] - Warning callback for null column values
|
|
380
|
+
* @param {Object} [deps={}] - Injectable dependencies (for testing)
|
|
381
|
+
* @param {Object} [deps.SenderClass] - QuestDB Sender class (default: @questdb/nodejs-client Sender)
|
|
382
|
+
* @param {Object} [deps.PgClientClass] - PostgreSQL Client class (default: pg.Client)
|
|
383
|
+
* @returns {Promise<Object>} Storage adapter with write, flush, close methods
|
|
384
|
+
*/
|
|
385
|
+
const createQuestDBStorage = async function ( assetClass, tablePrefix, options, deps = {} ) {
|
|
386
|
+
const {
|
|
387
|
+
ilpUrl = ENV_VARS.questdbIlpUrl,
|
|
388
|
+
pgUrl = ENV_VARS.questdbPgUrl,
|
|
389
|
+
flushMode = ENV_VARS.questdbFlushMode,
|
|
390
|
+
idleFlushAfterMs = ENV_VARS.questdbIdleFlushAfterMs,
|
|
391
|
+
idleFlushCheckMs = ENV_VARS.questdbIdleFlushCheckMs,
|
|
392
|
+
autoFlushRows = ENV_VARS.questdbAutoFlushRows,
|
|
393
|
+
autoFlushIntervalMs = ENV_VARS.questdbAutoFlushIntervalMs,
|
|
394
|
+
maxBufSize = ENV_VARS.questdbMaxBufSize,
|
|
395
|
+
retryTimeout = ENV_VARS.questdbRetryTimeout,
|
|
396
|
+
partitionBy = 'DAY',
|
|
397
|
+
onWarning,
|
|
398
|
+
onDeliveryFailure
|
|
399
|
+
} = options;
|
|
400
|
+
|
|
401
|
+
// Runtime validation — required from either DSL config or ENV_VARS.
|
|
402
|
+
// Per ADR-018, setup-time throws carry classified err.code.
|
|
403
|
+
if ( !ilpUrl ) {
|
|
404
|
+
const err = new Error( 'WinkComposer/questdb: ilpUrl required — set in .storage() config or QUESTDB_ILP_URL env var' );
|
|
405
|
+
err.code = 'INVALID_CONFIG';
|
|
406
|
+
throw err;
|
|
407
|
+
}
|
|
408
|
+
if ( !pgUrl ) {
|
|
409
|
+
const err = new Error( 'WinkComposer/questdb: pgUrl required — set in .storage() config or QUESTDB_PG_URL env var' );
|
|
410
|
+
err.code = 'INVALID_CONFIG';
|
|
411
|
+
throw err;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Injectable dependencies with defaults
|
|
415
|
+
const {
|
|
416
|
+
SenderClass = Sender,
|
|
417
|
+
PgClientClass = pg.Client
|
|
418
|
+
} = deps;
|
|
419
|
+
|
|
420
|
+
// Build persist plans (pre-compiled closures)
|
|
421
|
+
const persistPlans = buildPersistPlans( assetClass, tablePrefix, { onWarning, onDeliveryFailure } );
|
|
422
|
+
|
|
423
|
+
// Ensure tables exist via PostgreSQL wire protocol
|
|
424
|
+
const [ pgHost, pgPort ] = pgUrl.split( ':' );
|
|
425
|
+
const pgClient = new PgClientClass( {
|
|
426
|
+
host: pgHost,
|
|
427
|
+
port: parseInt( pgPort, 10 ),
|
|
428
|
+
database: ENV_VARS.questdbDatabase,
|
|
429
|
+
user: ENV_VARS.questdbUser,
|
|
430
|
+
password: ENV_VARS.questdbPassword
|
|
431
|
+
} );
|
|
432
|
+
|
|
433
|
+
// Wrap the PostgreSQL connect so the failure carries a classified
|
|
434
|
+
// `err.code` an operator can route on. The ADR-018 error vocabulary
|
|
435
|
+
// mandates one split here: an endpoint that does not answer (a Node syscall
|
|
436
|
+
// code like ECONNREFUSED — see NETWORK_ERROR_CODES above) is
|
|
437
|
+
// TRANSPORT_UNREACHABLE, because the remediation is "check the
|
|
438
|
+
// network, the firewall, whether the service is running" — the
|
|
439
|
+
// connection string may be perfectly fine. Everything else — an
|
|
440
|
+
// auth failure (the host answered), a protocol error, an
|
|
441
|
+
// unclassified throw — stays INVALID_CONFIG: fix the supplied
|
|
442
|
+
// config. The underlying error is preserved via `err.cause` for
|
|
443
|
+
// diagnostics either way.
|
|
444
|
+
try {
|
|
445
|
+
await pgClient.connect();
|
|
446
|
+
} catch ( connErr ) {
|
|
447
|
+
const err = new Error(
|
|
448
|
+
`WinkComposer/questdb: could not connect to PostgreSQL at ${pgUrl} — ${connErr.message}`
|
|
449
|
+
);
|
|
450
|
+
err.code = NETWORK_ERROR_CODES.has( connErr.code ) ?
|
|
451
|
+
'TRANSPORT_UNREACHABLE' :
|
|
452
|
+
'INVALID_CONFIG';
|
|
453
|
+
err.cause = connErr;
|
|
454
|
+
throw err;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
try {
|
|
458
|
+
await ensureTables( pgClient, assetClass, tablePrefix, { partitionBy } );
|
|
459
|
+
} finally {
|
|
460
|
+
await pgClient.end();
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// Create ILP sender
|
|
464
|
+
const senderConfig = buildSenderConfig( {
|
|
465
|
+
ilpUrl,
|
|
466
|
+
flushMode,
|
|
467
|
+
autoFlushRows,
|
|
468
|
+
autoFlushIntervalMs,
|
|
469
|
+
maxBufSize,
|
|
470
|
+
retryTimeout
|
|
471
|
+
} );
|
|
472
|
+
|
|
473
|
+
// fromConfig returns a Promise that resolves to a connected sender
|
|
474
|
+
const sender = await SenderClass.fromConfig( senderConfig );
|
|
475
|
+
|
|
476
|
+
// Buffer state — `bufferedRows` counts rows awaiting the NEXT flush.
|
|
477
|
+
// The client's flush() copies its rows out of the buffer synchronously
|
|
478
|
+
// (copy-out) before any network I/O, so the moment a flush is called
|
|
479
|
+
// its rows are no longer buffered — they move to the in-flight tally
|
|
480
|
+
// below. Conflating the two quantities in one counter let shutdown
|
|
481
|
+
// report clean over a hung idle flush.
|
|
482
|
+
let lastWriteTime = 0;
|
|
483
|
+
let idleFlushTimer = null;
|
|
484
|
+
let bufferedRows = 0;
|
|
485
|
+
|
|
486
|
+
// Rows inside flush copies that have not settled yet, plus the flush
|
|
487
|
+
// promises carrying them. getPressure() adds this to bufferedRows (a
|
|
488
|
+
// hung flush is undelivered data and must read as pressure), and
|
|
489
|
+
// shutdown() settles these entries instead of firing a blind flush
|
|
490
|
+
// at a buffer the copy-out already emptied.
|
|
491
|
+
let inFlightRows = 0;
|
|
492
|
+
const inFlightFlushes = new Set();
|
|
493
|
+
|
|
494
|
+
// Shutdown outcome, latched on the first call. A lossy shutdown's
|
|
495
|
+
// failed flush already emptied the buffer via copy-out, so a re-run
|
|
496
|
+
// would find nothing to flush and resolve clean — contradicting the
|
|
497
|
+
// recorded loss. Every caller gets the first call's promise instead.
|
|
498
|
+
let shutdownPromise = null;
|
|
499
|
+
|
|
500
|
+
// Health state — drives getHealth()'s status/connected derivation.
|
|
501
|
+
// shuttingDown flips true at the start of shutdown(); never resets.
|
|
502
|
+
// consecutiveWriteErrors increments on every write() catch and resets to 0
|
|
503
|
+
// on every successful persist plan call (recovery). Pure derived signal —
|
|
504
|
+
// no extra hot-path allocation; just two integer/boolean operations.
|
|
505
|
+
let shuttingDown = false;
|
|
506
|
+
let consecutiveWriteErrors = 0;
|
|
507
|
+
|
|
508
|
+
// One idle flush in flight at a time. Against an unreachable server a
|
|
509
|
+
// flush can never settle (the client's retry loop — see the mid-row
|
|
510
|
+
// recovery section in the file header); without this guard every check
|
|
511
|
+
// interval would start ANOTHER stuck flush. A hung flush leaves the
|
|
512
|
+
// flag set, the buffer keeps filling, and the condition surfaces as
|
|
513
|
+
// rising pressure in getHealth() — visible, not multiplied.
|
|
514
|
+
let idleFlushInFlight = false;
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Registers a flush the moment it is called. Copy-out means the rows
|
|
518
|
+
* leave the buffer NOW, delivered or not, so the caller hands them
|
|
519
|
+
* over in the same breath. Settlement — either way — removes them
|
|
520
|
+
* from the in-flight tally: resolved means delivered; rejected means
|
|
521
|
+
* lost, and loss REPORTING stays at the call site (the idle flush
|
|
522
|
+
* logs or routes to onDeliveryFailure, recovery routes to
|
|
523
|
+
* onDeliveryFailure, shutdown throws classified).
|
|
524
|
+
*
|
|
525
|
+
* @param {Promise} flushPromise - the just-fired sender.flush()
|
|
526
|
+
* @param {number} rows - row count the flush copy carries
|
|
527
|
+
* @returns {{promise: Promise, rows: number}} the tracked entry
|
|
528
|
+
*/
|
|
529
|
+
const trackFlush = function ( flushPromise, rows ) {
|
|
530
|
+
const entry = { promise: flushPromise, rows };
|
|
531
|
+
inFlightRows += rows;
|
|
532
|
+
inFlightFlushes.add( entry );
|
|
533
|
+
const settle = function () {
|
|
534
|
+
inFlightFlushes.delete( entry );
|
|
535
|
+
inFlightRows -= rows;
|
|
536
|
+
};
|
|
537
|
+
flushPromise.then( settle, settle );
|
|
538
|
+
return entry;
|
|
539
|
+
}; // trackFlush()
|
|
540
|
+
|
|
541
|
+
/**
|
|
542
|
+
* Check and flush if idle for idleFlushAfterMs.
|
|
543
|
+
*
|
|
544
|
+
* Runs in BOTH modes (see longer comment block at the setInterval call
|
|
545
|
+
* below). The rows move to the in-flight tally the moment the flush
|
|
546
|
+
* fires; a successful settle clears them, which also bounds the
|
|
547
|
+
* worst-case pressure-counter lag in auto mode (when QuestDB's silent
|
|
548
|
+
* `auto_flush_interval` trigger fires without us observing it, this
|
|
549
|
+
* is the self-healing point).
|
|
550
|
+
*/
|
|
551
|
+
const checkIdleFlush = async function () {
|
|
552
|
+
if ( idleFlushInFlight || bufferedRows === 0 ) {
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const now = Date.now();
|
|
557
|
+
const idleTime = now - lastWriteTime;
|
|
558
|
+
|
|
559
|
+
if ( idleTime >= idleFlushAfterMs ) {
|
|
560
|
+
idleFlushInFlight = true;
|
|
561
|
+
const rows = bufferedRows;
|
|
562
|
+
bufferedRows = 0;
|
|
563
|
+
const entry = trackFlush( sender.flush(), rows );
|
|
564
|
+
try {
|
|
565
|
+
await entry.promise;
|
|
566
|
+
} catch ( err ) {
|
|
567
|
+
// Copy-out: the failed flush carried the rows with it —
|
|
568
|
+
// the buffer no longer holds them and a retry cannot
|
|
569
|
+
// resend them. They are lost, and this is the report
|
|
570
|
+
// (the settle handler already cleared them from
|
|
571
|
+
// pressure). Same routing policy as the recovery flush:
|
|
572
|
+
// the caller owns the response when it asked to.
|
|
573
|
+
if ( onDeliveryFailure ) {
|
|
574
|
+
onDeliveryFailure( err, { idleFlush: true, rowsLost: rows } );
|
|
575
|
+
} else {
|
|
576
|
+
console.error(
|
|
577
|
+
`WinkComposer/questdb: idle flush failed; ${rows} buffered row(s) lost: ${err.message}`
|
|
578
|
+
);
|
|
579
|
+
}
|
|
580
|
+
} finally {
|
|
581
|
+
idleFlushInFlight = false;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
};
|
|
585
|
+
|
|
586
|
+
// Start idle flush timer as safety net for "data stopped flowing" case.
|
|
587
|
+
// QuestDB's auto_flush_interval only checks elapsed time when NEW data is added -
|
|
588
|
+
// it is NOT a real background timer. If no more data arrives, the buffer sits
|
|
589
|
+
// indefinitely. QuestDB docs recommend: "implement your own timer-based logic."
|
|
590
|
+
// See: https://py-questdb-client.readthedocs.io/en/stable/conf.html
|
|
591
|
+
//
|
|
592
|
+
// This timer runs in BOTH modes:
|
|
593
|
+
// - Auto mode: Safety net for when data stops flowing (QuestDB won't flush on its own)
|
|
594
|
+
// - Manual mode: Primary flush mechanism (QuestDB auto-flush is disabled)
|
|
595
|
+
idleFlushTimer = setInterval( checkIdleFlush, idleFlushCheckMs );
|
|
596
|
+
// Don't block process exit
|
|
597
|
+
idleFlushTimer.unref();
|
|
598
|
+
|
|
599
|
+
// ========================================================================
|
|
600
|
+
// STORAGE INTERFACE
|
|
601
|
+
// ========================================================================
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* Cancels a half-written ILP row after a mid-row throw, so the NEXT write
|
|
605
|
+
* starts clean (see "Mid-row recovery" in the file header for the full
|
|
606
|
+
* account and the upstream issue link).
|
|
607
|
+
*
|
|
608
|
+
* flush() copies every completed row out of the buffer synchronously and
|
|
609
|
+
* sends them in the background; the unfinished row stays behind. reset()
|
|
610
|
+
* then clears the buffer — only the broken stub remains at that point —
|
|
611
|
+
* and lowers the client's row-in-progress flags. Net effect: the broken
|
|
612
|
+
* row vanishes, every good row is on its way, the sender accepts the
|
|
613
|
+
* next write.
|
|
614
|
+
*
|
|
615
|
+
* The early flush carries real data. If its send ultimately fails (after
|
|
616
|
+
* the client's own retries), that loss is reported like any other
|
|
617
|
+
* delivery failure: routed to `onDeliveryFailure` when provided,
|
|
618
|
+
* otherwise thrown as a classified unhandled rejection — the same policy
|
|
619
|
+
* as persist-plan's sender.at() failure path. Never silent.
|
|
620
|
+
*/
|
|
621
|
+
const recoverSender = function () {
|
|
622
|
+
try {
|
|
623
|
+
// flush() is an async function: it can never throw synchronously,
|
|
624
|
+
// and the rows it sends live in its own copy of the buffer.
|
|
625
|
+
// Tracked like every flush, so shutdown settles it and its
|
|
626
|
+
// rows stay visible as pressure until it settles.
|
|
627
|
+
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
628
|
+
sender.reset();
|
|
629
|
+
bufferedRows = 0;
|
|
630
|
+
|
|
631
|
+
entry.promise.catch( ( flushErr ) => {
|
|
632
|
+
if ( onDeliveryFailure ) {
|
|
633
|
+
onDeliveryFailure( flushErr, { recovery: true } );
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
const failure = new Error(
|
|
637
|
+
`WinkComposer/questdb: recovery flush failed after a mid-row write error: ${flushErr.message}. ` +
|
|
638
|
+
'Completed rows in that batch were dropped. Provide an `onDeliveryFailure` callback ' +
|
|
639
|
+
'in the storage config to handle these explicitly.'
|
|
640
|
+
);
|
|
641
|
+
failure.code = 'DELIVERY_FAILED';
|
|
642
|
+
failure.cause = flushErr;
|
|
643
|
+
throw failure;
|
|
644
|
+
} );
|
|
645
|
+
} catch ( recoveryErr ) {
|
|
646
|
+
// Defensive: with the 4.2.0 client neither call can throw here
|
|
647
|
+
// (flush is async, reset is trivial buffer bookkeeping). If a
|
|
648
|
+
// future client changes that, the sender may stay wedged — but
|
|
649
|
+
// write() must still return its classified result rather than
|
|
650
|
+
// throw (ADR-018: the hot path never throws), and the
|
|
651
|
+
// failure must be visible.
|
|
652
|
+
console.error( `WinkComposer/questdb: sender recovery failed: ${recoveryErr.message}` );
|
|
653
|
+
}
|
|
654
|
+
}; // recoverSender()
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* Write a message to QuestDB for a given insightType.
|
|
658
|
+
*
|
|
659
|
+
* Sync hot-path return per ADR-018 / ADR-013:
|
|
660
|
+
* - `RESULT_OK` (singleton) on successful enqueue.
|
|
661
|
+
* - `{ ok: false, error: { code: 'INVALID_INSIGHT_TYPE', message } }`
|
|
662
|
+
* when the insightType has no persist plan (config error).
|
|
663
|
+
* - `{ ok: false, error: { code: 'SEND_FAILED', message } }` when the
|
|
664
|
+
* persist plan throws (sender method failure: ILP buffer overflow,
|
|
665
|
+
* type coercion failure, etc.).
|
|
666
|
+
*
|
|
667
|
+
* Never throws. Async tryFlush rejections inside `sender.at()` are
|
|
668
|
+
* contained at the call site in `persist-plan.js` (see file header).
|
|
669
|
+
*
|
|
670
|
+
* @param {string} insightType - SignalType name (must exist in assetClass)
|
|
671
|
+
* @param {Object} message - Message with column values
|
|
672
|
+
* @param {string} partitionId - Partition identifier (stored as SYMBOL)
|
|
673
|
+
* @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
|
|
674
|
+
*/
|
|
675
|
+
const write = function ( insightType, message, partitionId ) {
|
|
676
|
+
if ( shuttingDown ) {
|
|
677
|
+
// The idle timer is already stopped and the final flush may
|
|
678
|
+
// have run: a row accepted now would have no flusher left.
|
|
679
|
+
// Refuse loudly instead of stranding it — uniform with the
|
|
680
|
+
// MQTT emitter's SHUTTING_DOWN refusal.
|
|
681
|
+
return {
|
|
682
|
+
ok: false,
|
|
683
|
+
error: {
|
|
684
|
+
code: 'SHUTTING_DOWN',
|
|
685
|
+
message: 'WinkComposer/questdb: write rejected — storage is shutting down'
|
|
686
|
+
}
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
const persistPlan = persistPlans[ insightType ];
|
|
691
|
+
|
|
692
|
+
if ( !persistPlan ) {
|
|
693
|
+
return {
|
|
694
|
+
ok: false,
|
|
695
|
+
error: {
|
|
696
|
+
code: 'INVALID_INSIGHT_TYPE',
|
|
697
|
+
message: `No persist plan for insightType '${insightType}'`
|
|
698
|
+
}
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
try {
|
|
703
|
+
// The plan reports whether it actually opened a row. A row
|
|
704
|
+
// skipped in phase 1 (bad designated timestamp) never touched
|
|
705
|
+
// the sender and must not count as buffered — it would inflate
|
|
706
|
+
// pressure and shutdown's dropped count.
|
|
707
|
+
const written = persistPlan( sender, message, partitionId );
|
|
708
|
+
consecutiveWriteErrors = 0; // recovery — health flips back to green
|
|
709
|
+
if ( written ) {
|
|
710
|
+
lastWriteTime = Date.now();
|
|
711
|
+
bufferedRows += 1;
|
|
712
|
+
// Auto mode: QuestDB triggers an internal flush on the sender.at()
|
|
713
|
+
// call that brings pendingRowCount to autoFlushRows (verified
|
|
714
|
+
// against @questdb/nodejs-client v4.2.0 dist/cjs/index.js
|
|
715
|
+
// tryFlush: pendingRowCount >= autoFlushRows). Mirror that reset
|
|
716
|
+
// so getPressure() reflects QuestDB's actual buffer state.
|
|
717
|
+
//
|
|
718
|
+
// Guarded on `autoFlushRows > 0` because the value is optional in
|
|
719
|
+
// ENV_VARS / config; when unset, QuestDB falls back to its own
|
|
720
|
+
// internal default which we cannot observe. In that case the
|
|
721
|
+
// heuristic does not fire and QuestDB contributes no pressure
|
|
722
|
+
// signal — `getPressure()` (below) returns 0.
|
|
723
|
+
if ( flushMode === 'auto' && autoFlushRows > 0 && bufferedRows >= autoFlushRows ) {
|
|
724
|
+
bufferedRows = 0;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
return RESULT_OK;
|
|
728
|
+
} catch ( err ) {
|
|
729
|
+
consecutiveWriteErrors += 1; // health degradation signal
|
|
730
|
+
// The throw may have left a half-written row in the sender;
|
|
731
|
+
// cancel it so this failure costs one row, not the rest of the
|
|
732
|
+
// run (ADR-018). Safe to call even when no row was open: flushing
|
|
733
|
+
// early is harmless and reset() on a consistent buffer is a no-op.
|
|
734
|
+
recoverSender();
|
|
735
|
+
return {
|
|
736
|
+
ok: false,
|
|
737
|
+
error: {
|
|
738
|
+
code: 'SEND_FAILED',
|
|
739
|
+
message: err.message
|
|
740
|
+
}
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
};
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* Flush pending rows to QuestDB.
|
|
747
|
+
* In auto mode, this is a no-op (QuestDB handles it).
|
|
748
|
+
* In manual mode, this forces an immediate flush. The rows move to
|
|
749
|
+
* the in-flight tally at the call (copy-out); a failure rejects to
|
|
750
|
+
* the caller exactly as before, and the settle handler keeps the
|
|
751
|
+
* pressure accounting straight either way.
|
|
752
|
+
*
|
|
753
|
+
* @returns {Promise<void>}
|
|
754
|
+
*/
|
|
755
|
+
const flush = async function () {
|
|
756
|
+
if ( flushMode === 'manual' && bufferedRows > 0 ) {
|
|
757
|
+
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
758
|
+
bufferedRows = 0;
|
|
759
|
+
await entry.promise;
|
|
760
|
+
}
|
|
761
|
+
// Auto mode: no-op, QuestDB handles flushing
|
|
762
|
+
};
|
|
763
|
+
|
|
764
|
+
/**
|
|
765
|
+
* Backpressure metric for the partition manager. Returns the buffer fill
|
|
766
|
+
* ratio in [0, 1] per ADR-018 (sync, O(1), allocation-free). Clamped
|
|
767
|
+
* to 1.0 — the codebase invariant for getPressure across adapters.
|
|
768
|
+
*
|
|
769
|
+
* Accuracy by mode:
|
|
770
|
+
* - Manual mode: exact (we own every flush).
|
|
771
|
+
* - Auto mode, heavy flow: exact (autoFlushRows heuristic reset matches).
|
|
772
|
+
* - Auto mode, slow flow + auto_flush_interval triggers: over-states by
|
|
773
|
+
* up to `idleFlushAfterMs`, self-heals via `checkIdleFlush`.
|
|
774
|
+
*
|
|
775
|
+
* Rows inside unsettled flush copies count as pressure: a hung flush
|
|
776
|
+
* is undelivered data, and hiding it is what let shutdown report
|
|
777
|
+
* clean over it. They leave the tally when their flush settles —
|
|
778
|
+
* delivered or reported lost.
|
|
779
|
+
*
|
|
780
|
+
* When `autoFlushRows` is not configured (left to QuestDB's internal
|
|
781
|
+
* default), we have no observable capacity reference and return
|
|
782
|
+
* 0 — QuestDB then contributes nothing to the flow's pressure signal.
|
|
783
|
+
* The time-based yield (ADR-024) and message-count safety nets remain
|
|
784
|
+
* effective.
|
|
785
|
+
*
|
|
786
|
+
* @returns {number} Pressure value in [0, 1]
|
|
787
|
+
*/
|
|
788
|
+
const getPressure = function () {
|
|
789
|
+
if ( !( autoFlushRows > 0 ) ) {
|
|
790
|
+
return 0;
|
|
791
|
+
}
|
|
792
|
+
return Math.min( 1.0, ( bufferedRows + inFlightRows ) / autoFlushRows );
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
/**
|
|
796
|
+
* Health snapshot for operator monitoring (uniform across sinks).
|
|
797
|
+
* Returns the ADR-018 health floor `{status, connected, ...}`, plus a
|
|
798
|
+
* small set of QuestDB-specific diagnostic fields.
|
|
799
|
+
*
|
|
800
|
+
* Status derivation (kept in code, not config — operator mental model is
|
|
801
|
+
* load-bearing institutional knowledge):
|
|
802
|
+
* - `red` if `!connected` (shutting down or sustained write failure)
|
|
803
|
+
* - `yellow` if `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD` OR
|
|
804
|
+
* `consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD`
|
|
805
|
+
* - `green` otherwise
|
|
806
|
+
*
|
|
807
|
+
* `connected` here is *derived* — QuestDB's ILP sender is fire-and-forget
|
|
808
|
+
* with no observable socket state, so we infer transport health from
|
|
809
|
+
* recent write success.
|
|
810
|
+
*
|
|
811
|
+
* @returns {{status: 'green'|'yellow'|'red', connected: boolean, pressure: number, consecutiveWriteErrors: number, bufferedRows: number, flushMode: string}}
|
|
812
|
+
*/
|
|
813
|
+
const getHealth = function () {
|
|
814
|
+
const pressure = getPressure();
|
|
815
|
+
const connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD;
|
|
816
|
+
|
|
817
|
+
let status;
|
|
818
|
+
if ( !connected ) {
|
|
819
|
+
status = 'red';
|
|
820
|
+
} else if ( pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD || consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD ) {
|
|
821
|
+
status = 'yellow';
|
|
822
|
+
} else {
|
|
823
|
+
status = 'green';
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
return {
|
|
827
|
+
// Required health floor (ADR-018)
|
|
828
|
+
status,
|
|
829
|
+
connected,
|
|
830
|
+
pressure,
|
|
831
|
+
// Adapter-specific diagnostics
|
|
832
|
+
consecutiveWriteErrors,
|
|
833
|
+
bufferedRows,
|
|
834
|
+
flushMode
|
|
835
|
+
};
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
/**
|
|
839
|
+
* Best-effort transport close on the lossy path: the loss report
|
|
840
|
+
* (the classified throw that follows) matters more than a close
|
|
841
|
+
* failure, which is only logged.
|
|
842
|
+
*/
|
|
843
|
+
const closeQuietly = function () {
|
|
844
|
+
return sender.close().catch( function ( closeErr ) {
|
|
845
|
+
console.error( `WinkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
|
|
846
|
+
} );
|
|
847
|
+
}; // closeQuietly()
|
|
848
|
+
|
|
849
|
+
/**
|
|
850
|
+
* The real shutdown body. `shutdown` below latches its promise so
|
|
851
|
+
* every caller — including re-entrant and post-failure callers —
|
|
852
|
+
* receives this one outcome.
|
|
853
|
+
*
|
|
854
|
+
* A clean resolve is a delivery statement (ADR-018): everything
|
|
855
|
+
* buffered OR in flight was delivered. Shutdown therefore settles
|
|
856
|
+
* every unsettled flush (idle, recovery) plus one final flush for
|
|
857
|
+
* whatever is still buffered, all raced against the caller's
|
|
858
|
+
* `{ timeout }` (ADR-018 drain-then-close). It never fires a blind flush at a
|
|
859
|
+
* buffer an earlier copy-out emptied — that is what let it report
|
|
860
|
+
* clean over a hung idle flush.
|
|
861
|
+
*
|
|
862
|
+
* On loss it rejects classified, `dropped: { count }` exact:
|
|
863
|
+
* - any awaited flush fails → `DELIVERY_FAILED`, first flush error
|
|
864
|
+
* on `cause`, count = rows on the flushes that failed;
|
|
865
|
+
* - the combined wait does not settle in time → `SHUTDOWN_TIMEOUT`,
|
|
866
|
+
* count = rows not confirmed delivered. A send against an
|
|
867
|
+
* unreachable server never settles (the client's retry loop — see
|
|
868
|
+
* the file header), so the bound is what keeps shutdown finite.
|
|
869
|
+
* `dropped` is a statement about THIS session: those rows were not
|
|
870
|
+
* confirmed delivered before close. An abandoned flush keeps
|
|
871
|
+
* retrying and may still land its rows later if the server
|
|
872
|
+
* recovers — the count is a floor on uncertainty, not a proof of
|
|
873
|
+
* loss.
|
|
874
|
+
*
|
|
875
|
+
* The transport close is attempted in both loss paths, but on the
|
|
876
|
+
* HTTP transport the client's `close()` is an empty function
|
|
877
|
+
* (verified against @questdb/nodejs-client 4.2.0), so nothing can
|
|
878
|
+
* abort an abandoned flush's retry timers from here; they keep the
|
|
879
|
+
* event loop alive. Process exit is the final backstop — in a flow,
|
|
880
|
+
* the shutdown manager's `SHUTDOWN_FORCE_TIMEOUT_MS` exit covers
|
|
881
|
+
* this. No timeout supplied = no enforcement (unbounded await),
|
|
882
|
+
* preserving direct-caller behavior.
|
|
883
|
+
*/
|
|
884
|
+
const doShutdown = async function ( timeout ) {
|
|
885
|
+
// Flip the health flag first so any concurrent getHealth() call
|
|
886
|
+
// immediately sees the shutdown and returns red/disconnected —
|
|
887
|
+
// and write() starts refusing new rows (SHUTTING_DOWN).
|
|
888
|
+
shuttingDown = true;
|
|
889
|
+
|
|
890
|
+
// Stop idle flush timer
|
|
891
|
+
if ( idleFlushTimer ) {
|
|
892
|
+
clearInterval( idleFlushTimer );
|
|
893
|
+
idleFlushTimer = null;
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
// Everything delivery still owes: flushes already in flight
|
|
897
|
+
// (their rows left the buffer at their call) plus one final
|
|
898
|
+
// flush for whatever is still buffered. Each wait records its
|
|
899
|
+
// outcome into the tallies below; the mapped promises never
|
|
900
|
+
// reject, so the only rejection the race can surface is the
|
|
901
|
+
// timeout itself.
|
|
902
|
+
let totalRows = 0;
|
|
903
|
+
let deliveredRows = 0;
|
|
904
|
+
let failedRows = 0;
|
|
905
|
+
let firstFailure = null;
|
|
906
|
+
const waits = [];
|
|
907
|
+
|
|
908
|
+
const awaitDelivery = function ( entry ) {
|
|
909
|
+
totalRows += entry.rows;
|
|
910
|
+
waits.push( entry.promise.then(
|
|
911
|
+
function () {
|
|
912
|
+
deliveredRows += entry.rows;
|
|
913
|
+
},
|
|
914
|
+
function ( err ) {
|
|
915
|
+
failedRows += entry.rows;
|
|
916
|
+
if ( !firstFailure ) {
|
|
917
|
+
firstFailure = err;
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
) );
|
|
921
|
+
}; // awaitDelivery()
|
|
922
|
+
|
|
923
|
+
inFlightFlushes.forEach( awaitDelivery );
|
|
924
|
+
if ( bufferedRows > 0 ) {
|
|
925
|
+
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
926
|
+
bufferedRows = 0;
|
|
927
|
+
awaitDelivery( entry );
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
if ( waits.length > 0 ) {
|
|
931
|
+
try {
|
|
932
|
+
await raceFlushTimeout( Promise.all( waits ), timeout );
|
|
933
|
+
} catch ( err ) {
|
|
934
|
+
await closeQuietly();
|
|
935
|
+
const dropped = totalRows - deliveredRows;
|
|
936
|
+
const timedOut = new Error(
|
|
937
|
+
`WinkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
|
|
938
|
+
);
|
|
939
|
+
timedOut.code = 'SHUTDOWN_TIMEOUT';
|
|
940
|
+
timedOut.dropped = { count: dropped };
|
|
941
|
+
throw timedOut;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
if ( failedRows > 0 ) {
|
|
945
|
+
await closeQuietly();
|
|
946
|
+
const failure = new Error(
|
|
947
|
+
`WinkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
|
|
948
|
+
);
|
|
949
|
+
failure.code = 'DELIVERY_FAILED';
|
|
950
|
+
failure.dropped = { count: failedRows };
|
|
951
|
+
failure.cause = firstFailure;
|
|
952
|
+
throw failure;
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
// Close sender
|
|
957
|
+
await sender.close();
|
|
958
|
+
}; // doShutdown()
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* Shutdown the storage adapter gracefully.
|
|
962
|
+
* Flushes pending data and closes connections.
|
|
963
|
+
*
|
|
964
|
+
* Called by wire-storages.shutdown() which is invoked during:
|
|
965
|
+
* - Pipeline shutdown (flowHandle.shutdown())
|
|
966
|
+
* - Process signal handlers (SIGINT, SIGTERM)
|
|
967
|
+
*
|
|
968
|
+
* The outcome is latched: the first call runs the shutdown, every
|
|
969
|
+
* later call returns the same promise. A lossy shutdown's failed
|
|
970
|
+
* flush already emptied the buffer (copy-out), so a re-run would
|
|
971
|
+
* find nothing to flush and resolve clean, contradicting the
|
|
972
|
+
* recorded loss. One consequence: the first caller's `{ timeout }`
|
|
973
|
+
* governs; a later caller's is ignored.
|
|
974
|
+
*
|
|
975
|
+
* @param {{timeout?: number}} [options]
|
|
976
|
+
* @returns {Promise<void>}
|
|
977
|
+
*/
|
|
978
|
+
const shutdown = function ( { timeout = 0 } = {} ) {
|
|
979
|
+
if ( !shutdownPromise ) {
|
|
980
|
+
shutdownPromise = doShutdown( timeout );
|
|
981
|
+
}
|
|
982
|
+
return shutdownPromise;
|
|
983
|
+
}; // shutdown()
|
|
984
|
+
|
|
985
|
+
return {
|
|
986
|
+
write,
|
|
987
|
+
flush,
|
|
988
|
+
shutdown,
|
|
989
|
+
getPressure,
|
|
990
|
+
getHealth,
|
|
991
|
+
// Expose for testing/debugging
|
|
992
|
+
_sender: sender,
|
|
993
|
+
_persistPlans: persistPlans
|
|
994
|
+
};
|
|
995
|
+
};
|
|
996
|
+
|
|
997
|
+
// ============================================================================
|
|
998
|
+
// CONFIG SCHEMA
|
|
999
|
+
// ============================================================================
|
|
1000
|
+
|
|
1001
|
+
/**
|
|
1002
|
+
* Configuration schema for QuestDB storage adapter validation.
|
|
1003
|
+
* Used by flow.storage() to validate config at DSL time.
|
|
1004
|
+
*
|
|
1005
|
+
* `_propertyNames` lists every accepted key — unknown keys throw at
|
|
1006
|
+
* DSL time (the validator's only unknown-key mechanism).
|
|
1007
|
+
* Two factory-visible keys are deliberately absent: `assetClass`
|
|
1008
|
+
* (wire-storages injects it from the flow's `.assetClass()` after DSL
|
|
1009
|
+
* validation, so a user-supplied value would be silently overwritten)
|
|
1010
|
+
* and `_deps` (direct-call test injection only; direct calls bypass
|
|
1011
|
+
* this schema entirely).
|
|
1012
|
+
*
|
|
1013
|
+
* @type {Object}
|
|
1014
|
+
*/
|
|
1015
|
+
const configSchema = {
|
|
1016
|
+
_propertyNames: [
|
|
1017
|
+
'ilpUrl',
|
|
1018
|
+
'pgUrl',
|
|
1019
|
+
'tablePrefix',
|
|
1020
|
+
'flushMode',
|
|
1021
|
+
'idleFlushAfterMs',
|
|
1022
|
+
'idleFlushCheckMs',
|
|
1023
|
+
'autoFlushRows',
|
|
1024
|
+
'autoFlushIntervalMs',
|
|
1025
|
+
'maxBufSize',
|
|
1026
|
+
'retryTimeout',
|
|
1027
|
+
'partitionBy',
|
|
1028
|
+
'onWarning',
|
|
1029
|
+
'onDeliveryFailure'
|
|
1030
|
+
],
|
|
1031
|
+
ilpUrl: {
|
|
1032
|
+
type: 'string',
|
|
1033
|
+
required: false,
|
|
1034
|
+
minLength: 1,
|
|
1035
|
+
error: 'ilpUrl must be a non-empty string (e.g., localhost:9000)'
|
|
1036
|
+
},
|
|
1037
|
+
pgUrl: {
|
|
1038
|
+
type: 'string',
|
|
1039
|
+
required: false,
|
|
1040
|
+
minLength: 1,
|
|
1041
|
+
error: 'pgUrl must be a non-empty string (e.g., localhost:8812)'
|
|
1042
|
+
},
|
|
1043
|
+
tablePrefix: {
|
|
1044
|
+
type: 'string',
|
|
1045
|
+
required: false,
|
|
1046
|
+
minLength: 1,
|
|
1047
|
+
error: 'tablePrefix must be a non-empty string (defaults to assetClass.name when omitted)'
|
|
1048
|
+
},
|
|
1049
|
+
flushMode: {
|
|
1050
|
+
type: 'string',
|
|
1051
|
+
required: false,
|
|
1052
|
+
validator: validators.oneOf( [ 'auto', 'manual' ] ),
|
|
1053
|
+
error: 'flushMode must be "auto" or "manual"'
|
|
1054
|
+
},
|
|
1055
|
+
idleFlushAfterMs: {
|
|
1056
|
+
type: 'number',
|
|
1057
|
+
required: false,
|
|
1058
|
+
validator: validators.positiveInteger,
|
|
1059
|
+
error: 'idleFlushAfterMs must be a positive integer'
|
|
1060
|
+
},
|
|
1061
|
+
idleFlushCheckMs: {
|
|
1062
|
+
type: 'number',
|
|
1063
|
+
required: false,
|
|
1064
|
+
validator: validators.positiveInteger,
|
|
1065
|
+
error: 'idleFlushCheckMs must be a positive integer'
|
|
1066
|
+
},
|
|
1067
|
+
autoFlushRows: {
|
|
1068
|
+
type: 'number',
|
|
1069
|
+
required: false,
|
|
1070
|
+
validator: validators.positiveInteger,
|
|
1071
|
+
error: 'autoFlushRows must be a positive integer'
|
|
1072
|
+
},
|
|
1073
|
+
autoFlushIntervalMs: {
|
|
1074
|
+
type: 'number',
|
|
1075
|
+
required: false,
|
|
1076
|
+
validator: validators.positiveInteger,
|
|
1077
|
+
error: 'autoFlushIntervalMs must be a positive integer'
|
|
1078
|
+
},
|
|
1079
|
+
maxBufSize: {
|
|
1080
|
+
type: 'number',
|
|
1081
|
+
required: false,
|
|
1082
|
+
validator: validators.positiveInteger,
|
|
1083
|
+
error: 'maxBufSize must be a positive integer'
|
|
1084
|
+
},
|
|
1085
|
+
retryTimeout: {
|
|
1086
|
+
type: 'number',
|
|
1087
|
+
required: false,
|
|
1088
|
+
validator: validators.positiveInteger,
|
|
1089
|
+
error: 'retryTimeout must be a positive integer'
|
|
1090
|
+
},
|
|
1091
|
+
partitionBy: {
|
|
1092
|
+
type: 'string',
|
|
1093
|
+
required: false,
|
|
1094
|
+
validator: validators.oneOf( [ 'NONE', 'HOUR', 'DAY', 'WEEK', 'MONTH', 'YEAR' ] ),
|
|
1095
|
+
error: 'partitionBy must be one of: NONE, HOUR, DAY, WEEK, MONTH, YEAR'
|
|
1096
|
+
},
|
|
1097
|
+
onWarning: {
|
|
1098
|
+
type: 'function',
|
|
1099
|
+
required: false,
|
|
1100
|
+
error: 'onWarning must be a function'
|
|
1101
|
+
},
|
|
1102
|
+
onDeliveryFailure: {
|
|
1103
|
+
type: 'function',
|
|
1104
|
+
required: false,
|
|
1105
|
+
error: 'onDeliveryFailure must be a function'
|
|
1106
|
+
}
|
|
1107
|
+
};
|
|
1108
|
+
|
|
1109
|
+
// ============================================================================
|
|
1110
|
+
// ADAPTER EXPORT
|
|
1111
|
+
// ============================================================================
|
|
1112
|
+
|
|
1113
|
+
/**
|
|
1114
|
+
* Adapter identifier — matched by the flow registry and used as the
|
|
1115
|
+
* prefix in error messages.
|
|
1116
|
+
* @type {string}
|
|
1117
|
+
*/
|
|
1118
|
+
const id = 'questdb';
|
|
1119
|
+
|
|
1120
|
+
/**
|
|
1121
|
+
* Crash-survival class per ADR-018. Rows accepted by `write()` sit in
|
|
1122
|
+
* the ILP client's in-process buffer until flushed; a crash loses every
|
|
1123
|
+
* un-flushed row. The declaration states exactly what the code does; a
|
|
1124
|
+
* local write-ahead log could raise the class later.
|
|
1125
|
+
* @type {string}
|
|
1126
|
+
*/
|
|
1127
|
+
const durabilityClass = 'in-memory';
|
|
1128
|
+
|
|
1129
|
+
// Capability declaration per ADR-018.
|
|
1130
|
+
//
|
|
1131
|
+
// QuestDB needs the asset class because (a) ensure-tables.js reads
|
|
1132
|
+
// `columns` and `insightTypes` to issue CREATE TABLE statements, and
|
|
1133
|
+
// (b) persist-plan.js reads the same to dispatch column writers and
|
|
1134
|
+
// pre-compile resolution-aware quantization. `name` drives the default
|
|
1135
|
+
// `tablePrefix` when the caller did not pass one.
|
|
1136
|
+
//
|
|
1137
|
+
// Declared `required: true` because there is no useful behaviour for
|
|
1138
|
+
// QuestDB without an assetClass — the wiring layer fails fast with
|
|
1139
|
+
// MISSING_ASSET_CLASS rather than calling `createStorage` and letting
|
|
1140
|
+
// the failure surface deeper.
|
|
1141
|
+
//
|
|
1142
|
+
// `fields` lists exactly the top-level fields read. The slicing is
|
|
1143
|
+
// top-level only; column-internal fields (`type`, `resolution`) are
|
|
1144
|
+
// read directly from the `columns` slice and validated by Layer 2
|
|
1145
|
+
// assertions inside `createStorage` below. A declarative shape for
|
|
1146
|
+
// column-internal capability is deferred until a second adapter
|
|
1147
|
+
// needs it.
|
|
1148
|
+
const semanticsRequirement = {
|
|
1149
|
+
assetClass: {
|
|
1150
|
+
required: true,
|
|
1151
|
+
fields: [ 'name', 'columns', 'insightTypes' ]
|
|
1152
|
+
}
|
|
1153
|
+
};
|
|
1154
|
+
|
|
1155
|
+
/**
|
|
1156
|
+
* Create QuestDB storage instance.
|
|
1157
|
+
*
|
|
1158
|
+
* @param {Object} config - Configuration object
|
|
1159
|
+
* @param {Object} config.assetClass - Asset class definition with columns and insightTypes
|
|
1160
|
+
* @param {string} config.tablePrefix - Prefix for table names (defaults to assetClass.name)
|
|
1161
|
+
* @param {string} config.ilpUrl - ILP endpoint (host:port)
|
|
1162
|
+
* @param {string} config.pgUrl - PostgreSQL endpoint (host:port)
|
|
1163
|
+
* @param {string} [config.flushMode='auto'] - 'auto' or 'manual'
|
|
1164
|
+
* @param {number} [config.idleFlushAfterMs=5000] - Idle time before flush (manual mode)
|
|
1165
|
+
* @param {number} [config.idleFlushCheckMs=1000] - Idle check interval (manual mode)
|
|
1166
|
+
* @param {number} [config.autoFlushRows] - Rows before auto-flush (auto mode)
|
|
1167
|
+
* @param {number} [config.autoFlushIntervalMs] - Time interval for auto-flush (auto mode)
|
|
1168
|
+
* @param {number} [config.maxBufSize] - Maximum buffer size
|
|
1169
|
+
* @param {number} [config.retryTimeout] - Retry timeout in ms
|
|
1170
|
+
* @param {string} [config.partitionBy='DAY'] - Table partition interval
|
|
1171
|
+
* @param {function} [config.onWarning] - Warning callback for null column values
|
|
1172
|
+
* @param {Object} [config._deps] - Injectable dependencies (for testing)
|
|
1173
|
+
* @returns {Promise<Object>} Storage instance with write, flush, shutdown methods
|
|
1174
|
+
*/
|
|
1175
|
+
const createStorage = function ( config ) {
|
|
1176
|
+
const { assetClass, tablePrefix, _deps, ...options } = config;
|
|
1177
|
+
|
|
1178
|
+
// Validate assetClass (injected by wire-storages.js from flow's .assetClass())
|
|
1179
|
+
// tablePrefix is auto-defaulted to assetClass.name by wire-storages.js
|
|
1180
|
+
if ( !assetClass ) {
|
|
1181
|
+
// Per ADR-018, setup-time throws carry classified err.code.
|
|
1182
|
+
// MISSING_ASSET_CLASS (not INVALID_CONFIG) because the operator
|
|
1183
|
+
// remediation differs — they need to add .assetClass() to the
|
|
1184
|
+
// flow definition, not edit env vars or storage config.
|
|
1185
|
+
const err = new Error(
|
|
1186
|
+
'WinkComposer/questdb: assetClass is required - add .assetClass(assetClassDef) to flow before .storage()'
|
|
1187
|
+
);
|
|
1188
|
+
err.code = 'MISSING_ASSET_CLASS';
|
|
1189
|
+
throw err;
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
// Adapter-side defensive validation of column-internal facts
|
|
1193
|
+
// QuestDB consumes. Per the ADR-018 column-internal facts
|
|
1194
|
+
// pattern: the universal semantics
|
|
1195
|
+
// schema validates top-level
|
|
1196
|
+
// structure (type required, type in COLUMN_TYPES) at load and
|
|
1197
|
+
// at .assetClass() time; this layer catches bypasses (e.g.,
|
|
1198
|
+
// direct createStorage call from test code) and adapter-specific
|
|
1199
|
+
// requirements like "float64 columns with declared resolution
|
|
1200
|
+
// must have a positive value." See `assert-columns.js` for the
|
|
1201
|
+
// full rationale.
|
|
1202
|
+
assertColumnFacts( assetClass );
|
|
1203
|
+
|
|
1204
|
+
// Default `tablePrefix` to the asset class name when the caller
|
|
1205
|
+
// did not supply one. This default used to live in wire-storages
|
|
1206
|
+
// and moved here so the wiring layer stays generic and each
|
|
1207
|
+
// adapter owns its own defaulting policy. Same observable
|
|
1208
|
+
// behaviour: callers passing an explicit `tablePrefix` see it
|
|
1209
|
+
// honoured; callers omitting it get `assetClass.name`.
|
|
1210
|
+
const effectiveTablePrefix = tablePrefix ?? assetClass.name;
|
|
1211
|
+
|
|
1212
|
+
// Returns Promise from createQuestDBStorage (already async)
|
|
1213
|
+
return createQuestDBStorage( assetClass, effectiveTablePrefix, options, _deps );
|
|
1214
|
+
}; // createStorage()
|
|
1215
|
+
|
|
1216
|
+
/**
|
|
1217
|
+
* QuestDB storage adapter for flow DSL integration — the default
|
|
1218
|
+
* aggregate ADR-018's module surface calls for. References the same
|
|
1219
|
+
* constants as the named exports; never
|
|
1220
|
+
* a second source of truth.
|
|
1221
|
+
*
|
|
1222
|
+
* Usage:
|
|
1223
|
+
* import questdbAdapter from './core/storage-manager/questdb/index.js';
|
|
1224
|
+
*
|
|
1225
|
+
* flow( 'myFlow' )
|
|
1226
|
+
* .assetClass( assetClassDef ) // wire-storages injects the slice
|
|
1227
|
+
* .storage( questdbAdapter, {
|
|
1228
|
+
* tablePrefix: 'myPrefix',
|
|
1229
|
+
* ilpUrl: 'localhost:9000',
|
|
1230
|
+
* pgUrl: 'localhost:8812'
|
|
1231
|
+
* } )
|
|
1232
|
+
*
|
|
1233
|
+
* @type {Object}
|
|
1234
|
+
*/
|
|
1235
|
+
const questdbAdapter = { id, configSchema, durabilityClass, semanticsRequirement, createStorage };
|
|
1236
|
+
|
|
1237
|
+
// ============================================================================
|
|
1238
|
+
// EXPORTS
|
|
1239
|
+
// ============================================================================
|
|
1240
|
+
|
|
1241
|
+
export default questdbAdapter;
|
|
1242
|
+
export {
|
|
1243
|
+
id,
|
|
1244
|
+
configSchema,
|
|
1245
|
+
durabilityClass,
|
|
1246
|
+
semanticsRequirement,
|
|
1247
|
+
createStorage,
|
|
1248
|
+
createQuestDBStorage,
|
|
1249
|
+
buildSenderConfig,
|
|
1250
|
+
questdbAdapter
|
|
1251
|
+
};
|