@winkjs/composer 0.0.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +42 -103
- package/package.json +53 -29
- package/src/composer.js +75 -0
- package/src/core/batch/aoo-encoder.js +148 -0
- package/src/core/batch/index.js +12 -0
- package/src/core/batch/sliding-window.js +91 -0
- package/src/core/codec/index.js +137 -0
- package/src/core/emitter-manager/mqtt/constants.js +102 -0
- package/src/core/emitter-manager/mqtt/emitter.js +780 -0
- package/src/core/emitter-manager/mqtt/index.js +190 -0
- package/src/core/emitter-manager/mqtt/mqtt-store.js +538 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +98 -0
- package/src/core/emitter-manager/terminal/formatters.js +160 -0
- package/src/core/emitter-manager/terminal/index.js +341 -0
- package/src/core/env-vars.js +245 -0
- package/src/core/mqtt-protocol.js +47 -0
- package/src/core/partition-manager/index.js +7 -0
- package/src/core/partition-manager/init.js +44 -0
- package/src/core/partition-manager/introspect.js +15 -0
- package/src/core/partition-manager/publish-to.js +6 -0
- package/src/core/partition-manager/recompute.js +5 -0
- package/src/core/partition-manager/reset.js +5 -0
- package/src/core/partition-manager/update.js +264 -0
- package/src/core/partition-manager/validate-flow.js +157 -0
- package/src/core/semantics/digest.js +128 -0
- package/src/core/semantics/index.js +83 -0
- package/src/core/semantics/loader.js +571 -0
- package/src/core/semantics/schemas/asset-class-schema.js +218 -0
- package/src/core/semantics/schemas/column-schema.js +651 -0
- package/src/core/semantics/schemas/enum-schema.js +153 -0
- package/src/core/semantics/schemas/index.js +51 -0
- package/src/core/semantics/warnings.js +141 -0
- package/src/core/shutdown-manager/index.js +289 -0
- package/src/core/source-manager/csv/index.js +171 -0
- package/src/core/source-manager/csv/start.js +456 -0
- package/src/core/source-manager/mqtt/client.js +406 -0
- package/src/core/source-manager/mqtt/constants.js +147 -0
- package/src/core/source-manager/mqtt/dedup.js +244 -0
- package/src/core/source-manager/mqtt/index.js +260 -0
- package/src/core/source-manager/mqtt/status.js +546 -0
- package/src/core/source-manager/test-harness/comparator.js +226 -0
- package/src/core/source-manager/test-harness/field-generator.js +85 -0
- package/src/core/source-manager/test-harness/fuzz.js +87 -0
- package/src/core/source-manager/test-harness/index.js +115 -0
- package/src/core/source-manager/test-harness/prng.js +57 -0
- package/src/core/source-manager/test-harness/start.js +232 -0
- package/src/core/source-manager/test-harness/validate.js +213 -0
- package/src/core/storage-manager/questdb/assert-columns.js +120 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +187 -0
- package/src/core/storage-manager/questdb/index.js +1251 -0
- package/src/core/storage-manager/questdb/persist-plan.js +488 -0
- package/src/core/storage-manager/questdb/writers.js +142 -0
- package/src/core/test-utils/sealed-record.js +80 -0
- package/src/core/test-utils/soak-signature.js +149 -0
- package/src/core/test-utils/tcp-proxy.js +106 -0
- package/src/core/tunable/helpers.js +254 -0
- package/src/core/tunable/index.js +87 -0
- package/src/core/utils/console-formatter/index.js +207 -0
- package/src/core/utils/create-smart-store-as/index.js +98 -0
- package/src/core/utils/create-smart-store-as/sanitize-field.js +46 -0
- package/src/core/utils/create-smart-store-as/smart-truncate.js +93 -0
- package/src/core/utils/flow/index.js +5 -0
- package/src/core/utils/flow/make-collision-checker.js +23 -0
- package/src/core/utils/flow/to-kebab.js +20 -0
- package/src/core/utils/half-life/index.js +111 -0
- package/src/core/utils/node/disable.js +11 -0
- package/src/core/utils/node/enable.js +11 -0
- package/src/core/utils/node/execute-triggers.js +40 -0
- package/src/core/utils/node/index.js +10 -0
- package/src/core/utils/node/pause.js +11 -0
- package/src/core/utils/node/populate-predicate-inputs.js +14 -0
- package/src/core/utils/node/publish-nan.js +19 -0
- package/src/core/utils/node/reset-predicate-inputs.js +11 -0
- package/src/core/utils/node/unpause.js +11 -0
- package/src/core/utils/node/validate-spec.js +22 -0
- package/src/core/utils/options/resolve-field-keyed.js +115 -0
- package/src/core/utils/quantize/index.js +75 -0
- package/src/core/utils/stats/formulas.js +67 -0
- package/src/core/utils/stats/index.js +3 -0
- package/src/core/utils/validate/composer-validators.js +141 -0
- package/src/core/utils/validate/helpers.js +206 -0
- package/src/core/utils/validate/index.js +12 -0
- package/src/core/utils/validate/type-validators.js +538 -0
- package/src/core/utils/validate/validate.js +107 -0
- package/src/core/utils/validate/validators.js +177 -0
- package/src/core/wiring/assert-handle.js +57 -0
- package/src/core/wiring/assert-module.js +63 -0
- package/src/core/wiring/index.js +5 -0
- package/src/core/wiring/node-type-to-module.js +17 -0
- package/src/core/wiring/wire-emitters.js +256 -0
- package/src/core/wiring/wire-linear-graph.js +25 -0
- package/src/core/wiring/wire-node.js +157 -0
- package/src/core/wiring/wire-semantics.js +173 -0
- package/src/core/wiring/wire-storages.js +344 -0
- package/src/flow/check-output-collisions.js +93 -0
- package/src/flow/consts.js +9 -0
- package/src/flow/deep-clone-spec.js +64 -0
- package/src/flow/driver.js +171 -0
- package/src/flow/expand-single-input-array.js +76 -0
- package/src/flow/expand-template-spec.js +93 -0
- package/src/flow/flow-api-schemas.js +300 -0
- package/src/flow/flow.js +658 -0
- package/src/flow/for-each.js +225 -0
- package/src/flow/get-signature-pattern.js +31 -0
- package/src/flow/get-target-array.js +46 -0
- package/src/flow/inspect.js +67 -0
- package/src/flow/load-node-module.js +81 -0
- package/src/flow/make-node-method.js +112 -0
- package/src/flow/resolve-pick-by-field.js +77 -0
- package/src/flow/resolve-tunables.js +76 -0
- package/src/flow/run.js +392 -0
- package/src/flow/serialize.js +161 -0
- package/src/flow/signatures-schema.js +222 -0
- package/src/flow/spec-builder.js +65 -0
- package/src/flow/validate-triggers.js +250 -0
- package/src/flow/validate.js +261 -0
- package/src/nodes/accumulate/index.js +21 -0
- package/src/nodes/accumulate/init.js +48 -0
- package/src/nodes/accumulate/introspect.js +123 -0
- package/src/nodes/accumulate/publish-to.js +34 -0
- package/src/nodes/accumulate/recompute.js +27 -0
- package/src/nodes/accumulate/reset.js +25 -0
- package/src/nodes/accumulate/update.js +42 -0
- package/src/nodes/appraise/calibrate.js +64 -0
- package/src/nodes/appraise/decision.js +66 -0
- package/src/nodes/appraise/deviation.js +140 -0
- package/src/nodes/appraise/index.js +23 -0
- package/src/nodes/appraise/init.js +165 -0
- package/src/nodes/appraise/integrate.js +45 -0
- package/src/nodes/appraise/introspect.js +352 -0
- package/src/nodes/appraise/publish-to.js +93 -0
- package/src/nodes/appraise/receptor.js +69 -0
- package/src/nodes/appraise/recompute.js +79 -0
- package/src/nodes/appraise/reset.js +53 -0
- package/src/nodes/appraise/update.js +130 -0
- package/src/nodes/archive/csv/index.js +6 -0
- package/src/nodes/archive/csv/init.js +36 -0
- package/src/nodes/archive/csv/load.js +70 -0
- package/src/nodes/archive/csv/reset.js +14 -0
- package/src/nodes/archive/csv/simple-parser.js +216 -0
- package/src/nodes/archive/csv/test.js +173 -0
- package/src/nodes/archive/csv/update.js +63 -0
- package/src/nodes/butterworth-filter/index.js +18 -0
- package/src/nodes/butterworth-filter/init.js +184 -0
- package/src/nodes/butterworth-filter/introspect.js +228 -0
- package/src/nodes/butterworth-filter/publish-to.js +21 -0
- package/src/nodes/butterworth-filter/recompute.js +13 -0
- package/src/nodes/butterworth-filter/reset.js +30 -0
- package/src/nodes/butterworth-filter/update.js +38 -0
- package/src/nodes/categorize/index.js +19 -0
- package/src/nodes/categorize/init.js +58 -0
- package/src/nodes/categorize/introspect.js +177 -0
- package/src/nodes/categorize/publish-to.js +29 -0
- package/src/nodes/categorize/recompute.js +14 -0
- package/src/nodes/categorize/reset.js +16 -0
- package/src/nodes/categorize/update.js +53 -0
- package/src/nodes/controller/index.js +9 -0
- package/src/nodes/controller/init.js +55 -0
- package/src/nodes/controller/introspect.js +91 -0
- package/src/nodes/controller/publish-to.js +8 -0
- package/src/nodes/controller/recompute.js +9 -0
- package/src/nodes/controller/reset.js +17 -0
- package/src/nodes/controller/update.js +63 -0
- package/src/nodes/diff/index.js +19 -0
- package/src/nodes/diff/init.js +37 -0
- package/src/nodes/diff/introspect.js +132 -0
- package/src/nodes/diff/publish-to.js +21 -0
- package/src/nodes/diff/recompute.js +12 -0
- package/src/nodes/diff/reset.js +12 -0
- package/src/nodes/diff/update.js +37 -0
- package/src/nodes/digest-moments/formulas.js +12 -0
- package/src/nodes/digest-moments/index.js +14 -0
- package/src/nodes/digest-moments/init.js +67 -0
- package/src/nodes/digest-moments/introspect.js +154 -0
- package/src/nodes/digest-moments/publish-to.js +34 -0
- package/src/nodes/digest-moments/recompute.js +14 -0
- package/src/nodes/digest-moments/reset.js +14 -0
- package/src/nodes/digest-moments/update.js +120 -0
- package/src/nodes/dwell-time-tracker/index.js +14 -0
- package/src/nodes/dwell-time-tracker/init.js +59 -0
- package/src/nodes/dwell-time-tracker/introspect.js +137 -0
- package/src/nodes/dwell-time-tracker/publish-to.js +22 -0
- package/src/nodes/dwell-time-tracker/recompute.js +8 -0
- package/src/nodes/dwell-time-tracker/reset.js +23 -0
- package/src/nodes/dwell-time-tracker/update.js +103 -0
- package/src/nodes/emit-if/helpers.js +94 -0
- package/src/nodes/emit-if/index.js +16 -0
- package/src/nodes/emit-if/init.js +67 -0
- package/src/nodes/emit-if/introspect.js +102 -0
- package/src/nodes/emit-if/publish-to.js +13 -0
- package/src/nodes/emit-if/recompute.js +12 -0
- package/src/nodes/emit-if/reset.js +27 -0
- package/src/nodes/emit-if/update.js +93 -0
- package/src/nodes/es-correlation/index.js +19 -0
- package/src/nodes/es-correlation/init.js +73 -0
- package/src/nodes/es-correlation/introspect.js +176 -0
- package/src/nodes/es-correlation/publish-to.js +32 -0
- package/src/nodes/es-correlation/recompute.js +56 -0
- package/src/nodes/es-correlation/reset.js +27 -0
- package/src/nodes/es-correlation/update.js +110 -0
- package/src/nodes/es-mean/index.js +20 -0
- package/src/nodes/es-mean/init.js +58 -0
- package/src/nodes/es-mean/introspect.js +129 -0
- package/src/nodes/es-mean/publish-to.js +19 -0
- package/src/nodes/es-mean/recompute.js +16 -0
- package/src/nodes/es-mean/reset.js +24 -0
- package/src/nodes/es-mean/update.js +69 -0
- package/src/nodes/es-pairwise-correlation/index.js +19 -0
- package/src/nodes/es-pairwise-correlation/init.js +98 -0
- package/src/nodes/es-pairwise-correlation/introspect.js +187 -0
- package/src/nodes/es-pairwise-correlation/publish-to.js +31 -0
- package/src/nodes/es-pairwise-correlation/recompute.js +69 -0
- package/src/nodes/es-pairwise-correlation/reset.js +29 -0
- package/src/nodes/es-pairwise-correlation/update.js +89 -0
- package/src/nodes/es-stats/index.js +21 -0
- package/src/nodes/es-stats/init.js +105 -0
- package/src/nodes/es-stats/introspect.js +157 -0
- package/src/nodes/es-stats/publish-to.js +42 -0
- package/src/nodes/es-stats/recompute.js +74 -0
- package/src/nodes/es-stats/reset.js +49 -0
- package/src/nodes/es-stats/update-envelope.js +44 -0
- package/src/nodes/es-stats/update-welford.js +64 -0
- package/src/nodes/es-stats/update.js +59 -0
- package/src/nodes/index.js +83 -0
- package/src/nodes/invert-flag/index.js +20 -0
- package/src/nodes/invert-flag/init.js +36 -0
- package/src/nodes/invert-flag/introspect.js +115 -0
- package/src/nodes/invert-flag/publish-to.js +23 -0
- package/src/nodes/invert-flag/recompute.js +12 -0
- package/src/nodes/invert-flag/reset.js +12 -0
- package/src/nodes/invert-flag/update.js +32 -0
- package/src/nodes/kalman1d/index.js +20 -0
- package/src/nodes/kalman1d/init.js +103 -0
- package/src/nodes/kalman1d/introspect.js +250 -0
- package/src/nodes/kalman1d/publish-to.js +34 -0
- package/src/nodes/kalman1d/recompute.js +23 -0
- package/src/nodes/kalman1d/reset.js +29 -0
- package/src/nodes/kalman1d/update.js +144 -0
- package/src/nodes/kernel/index.js +18 -0
- package/src/nodes/kernel/init.js +66 -0
- package/src/nodes/kernel/introspect.js +163 -0
- package/src/nodes/kernel/presets.js +74 -0
- package/src/nodes/kernel/publish-to.js +25 -0
- package/src/nodes/kernel/recompute.js +14 -0
- package/src/nodes/kernel/reset.js +20 -0
- package/src/nodes/kernel/update.js +69 -0
- package/src/nodes/lag/index.js +21 -0
- package/src/nodes/lag/init.js +87 -0
- package/src/nodes/lag/introspect.js +202 -0
- package/src/nodes/lag/publish-to.js +64 -0
- package/src/nodes/lag/recompute.js +33 -0
- package/src/nodes/lag/reset.js +49 -0
- package/src/nodes/lag/update.js +133 -0
- package/src/nodes/median3/index.js +19 -0
- package/src/nodes/median3/init.js +40 -0
- package/src/nodes/median3/introspect.js +116 -0
- package/src/nodes/median3/publish-to.js +21 -0
- package/src/nodes/median3/recompute.js +12 -0
- package/src/nodes/median3/reset.js +15 -0
- package/src/nodes/median3/update.js +58 -0
- package/src/nodes/moments-digest/copy-from.js +12 -0
- package/src/nodes/moments-digest/flush.js +17 -0
- package/src/nodes/moments-digest/index.js +132 -0
- package/src/nodes/moments-digest/init.js +84 -0
- package/src/nodes/moments-digest/introspect.js +122 -0
- package/src/nodes/moments-digest/prelude.js +28 -0
- package/src/nodes/moments-digest/publish-to.js +31 -0
- package/src/nodes/moments-digest/recompute.js +9 -0
- package/src/nodes/moments-digest/reset.js +20 -0
- package/src/nodes/moments-digest/update.js +182 -0
- package/src/nodes/node-names.js +67 -0
- package/src/nodes/page-hinkley/index.js +20 -0
- package/src/nodes/page-hinkley/init.js +66 -0
- package/src/nodes/page-hinkley/introspect.js +193 -0
- package/src/nodes/page-hinkley/publish-to.js +28 -0
- package/src/nodes/page-hinkley/recompute.js +14 -0
- package/src/nodes/page-hinkley/reset.js +23 -0
- package/src/nodes/page-hinkley/update.js +105 -0
- package/src/nodes/pass-if/index.js +21 -0
- package/src/nodes/pass-if/init.js +37 -0
- package/src/nodes/pass-if/introspect.js +93 -0
- package/src/nodes/pass-if/publish-to.js +15 -0
- package/src/nodes/pass-if/recompute.js +15 -0
- package/src/nodes/pass-if/reset.js +18 -0
- package/src/nodes/pass-if/update.js +37 -0
- package/src/nodes/persist-if/helpers.js +111 -0
- package/src/nodes/persist-if/index.js +9 -0
- package/src/nodes/persist-if/init.js +98 -0
- package/src/nodes/persist-if/introspect.js +106 -0
- package/src/nodes/persist-if/publish-to.js +21 -0
- package/src/nodes/persist-if/recompute.js +20 -0
- package/src/nodes/persist-if/reset.js +35 -0
- package/src/nodes/persist-if/update.js +112 -0
- package/src/nodes/persistence-check/index.js +14 -0
- package/src/nodes/persistence-check/init.js +38 -0
- package/src/nodes/persistence-check/introspect.js +144 -0
- package/src/nodes/persistence-check/publish-to.js +11 -0
- package/src/nodes/persistence-check/recompute.js +7 -0
- package/src/nodes/persistence-check/reset.js +12 -0
- package/src/nodes/persistence-check/update.js +57 -0
- package/src/nodes/process-index/index.js +15 -0
- package/src/nodes/process-index/init.js +48 -0
- package/src/nodes/process-index/introspect.js +182 -0
- package/src/nodes/process-index/publish-to.js +29 -0
- package/src/nodes/process-index/recompute.js +8 -0
- package/src/nodes/process-index/reset.js +8 -0
- package/src/nodes/process-index/update.js +75 -0
- package/src/nodes/ratio/index.js +14 -0
- package/src/nodes/ratio/init.js +37 -0
- package/src/nodes/ratio/introspect.js +155 -0
- package/src/nodes/ratio/publish-to.js +15 -0
- package/src/nodes/ratio/recompute.js +7 -0
- package/src/nodes/ratio/reset.js +7 -0
- package/src/nodes/ratio/update.js +47 -0
- package/src/nodes/sanitize/index.js +82 -0
- package/src/nodes/sanitize/init.js +82 -0
- package/src/nodes/sanitize/introspect.js +194 -0
- package/src/nodes/sanitize/publish-to.js +15 -0
- package/src/nodes/sanitize/recompute.js +8 -0
- package/src/nodes/sanitize/reset.js +14 -0
- package/src/nodes/sanitize/update.js +102 -0
- package/src/nodes/spike-guard/index.js +14 -0
- package/src/nodes/spike-guard/init.js +47 -0
- package/src/nodes/spike-guard/introspect.js +130 -0
- package/src/nodes/spike-guard/publish-to.js +35 -0
- package/src/nodes/spike-guard/recompute.js +14 -0
- package/src/nodes/spike-guard/reset.js +19 -0
- package/src/nodes/spike-guard/update.js +89 -0
- package/src/nodes/sse-emitter/index.js +7 -0
- package/src/nodes/sse-emitter/init.js +105 -0
- package/src/nodes/sse-emitter/introspect.js +134 -0
- package/src/nodes/sse-emitter/publish-to.js +6 -0
- package/src/nodes/sse-emitter/recompute.js +16 -0
- package/src/nodes/sse-emitter/reset.js +19 -0
- package/src/nodes/sse-emitter/update.js +69 -0
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +36 -0
- package/src/nodes/sse-emitter/utils/build-message.js +31 -0
- package/src/nodes/sse-emitter/utils/connection-manager.js +45 -0
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +20 -0
- package/src/nodes/sse-emitter/utils/health-monitor.js +38 -0
- package/src/nodes/sse-emitter/utils/qos-handler.js +41 -0
- package/src/nodes/state-change-detector/helpers.js +35 -0
- package/src/nodes/state-change-detector/index.js +12 -0
- package/src/nodes/state-change-detector/init.js +56 -0
- package/src/nodes/state-change-detector/introspect.js +146 -0
- package/src/nodes/state-change-detector/publish-to.js +20 -0
- package/src/nodes/state-change-detector/recompute.js +6 -0
- package/src/nodes/state-change-detector/reset.js +20 -0
- package/src/nodes/state-change-detector/update.js +97 -0
- package/src/nodes/sw-stats/index.js +14 -0
- package/src/nodes/sw-stats/init.js +51 -0
- package/src/nodes/sw-stats/introspect.js +131 -0
- package/src/nodes/sw-stats/publish-to.js +45 -0
- package/src/nodes/sw-stats/recompute.js +33 -0
- package/src/nodes/sw-stats/reset.js +15 -0
- package/src/nodes/sw-stats/update.js +39 -0
- package/src/nodes/swing-watch/index.js +17 -0
- package/src/nodes/swing-watch/init.js +144 -0
- package/src/nodes/swing-watch/introspect.js +188 -0
- package/src/nodes/swing-watch/publish-to.js +53 -0
- package/src/nodes/swing-watch/recompute.js +22 -0
- package/src/nodes/swing-watch/reset.js +55 -0
- package/src/nodes/swing-watch/update.js +767 -0
- package/src/nodes/tally/index.js +15 -0
- package/src/nodes/tally/init.js +32 -0
- package/src/nodes/tally/introspect.js +116 -0
- package/src/nodes/tally/publish-to.js +28 -0
- package/src/nodes/tally/recompute.js +10 -0
- package/src/nodes/tally/reset.js +10 -0
- package/src/nodes/tally/update.js +44 -0
- package/src/nodes/threshold/index.js +14 -0
- package/src/nodes/threshold/init.js +121 -0
- package/src/nodes/threshold/introspect.js +192 -0
- package/src/nodes/threshold/publish-to.js +11 -0
- package/src/nodes/threshold/recompute.js +8 -0
- package/src/nodes/threshold/reset.js +14 -0
- package/src/nodes/threshold/update.js +54 -0
- package/src/nodes/transform/helpers.js +39 -0
- package/src/nodes/transform/index.js +15 -0
- package/src/nodes/transform/init.js +35 -0
- package/src/nodes/transform/introspect.js +118 -0
- package/src/nodes/transform/publish-to.js +23 -0
- package/src/nodes/transform/recompute.js +12 -0
- package/src/nodes/transform/reset.js +11 -0
- package/src/nodes/transform/update.js +32 -0
- package/src/nodes/trend/compute-confidence.js +112 -0
- package/src/nodes/trend/index.js +14 -0
- package/src/nodes/trend/init.js +81 -0
- package/src/nodes/trend/introspect.js +151 -0
- package/src/nodes/trend/publish-to.js +22 -0
- package/src/nodes/trend/recompute.js +15 -0
- package/src/nodes/trend/reset.js +26 -0
- package/src/nodes/trend/update.js +93 -0
- package/src/nodes/tw-stats/copy-from.js +16 -0
- package/src/nodes/tw-stats/flush.js +12 -0
- package/src/nodes/tw-stats/index.js +41 -0
- package/src/nodes/tw-stats/init.js +92 -0
- package/src/nodes/tw-stats/introspect.js +171 -0
- package/src/nodes/tw-stats/publish-to.js +108 -0
- package/src/nodes/tw-stats/recompute.js +9 -0
- package/src/nodes/tw-stats/reset.js +16 -0
- package/src/nodes/tw-stats/update.js +113 -0
- package/src/nodes/unbalance/index.js +15 -0
- package/src/nodes/unbalance/init.js +59 -0
- package/src/nodes/unbalance/introspect.js +167 -0
- package/src/nodes/unbalance/publish-to.js +52 -0
- package/src/nodes/unbalance/recompute.js +11 -0
- package/src/nodes/unbalance/reset.js +10 -0
- package/src/nodes/unbalance/update.js +83 -0
- package/src/nodes/vector-distance/index.js +14 -0
- package/src/nodes/vector-distance/init.js +54 -0
- package/src/nodes/vector-distance/introspect.js +149 -0
- package/src/nodes/vector-distance/publish-to.js +35 -0
- package/src/nodes/vector-distance/recompute.js +11 -0
- package/src/nodes/vector-distance/reset.js +20 -0
- package/src/nodes/vector-distance/update.js +139 -0
- package/src/nodes/winnow/index.js +12 -0
- package/src/nodes/winnow/init.js +82 -0
- package/src/nodes/winnow/introspect.js +253 -0
- package/src/nodes/winnow/publish-to.js +51 -0
- package/src/nodes/winnow/recompute.js +14 -0
- package/src/nodes/winnow/reset.js +34 -0
- package/src/nodes/winnow/update.js +203 -0
- package/src/tools/training/features.js +227 -0
- package/src/tools/training/index.js +21 -0
- package/src/tools/training/logistic.js +238 -0
- package/src/tools/training/metrics.js +126 -0
- package/src/tools/training/scale.js +239 -0
- package/src/tools/training/validate.js +103 -0
- package/src/windowing/count-sliding/create.js +16 -0
- package/src/windowing/count-sliding/get-reverse-chronological-index.js +19 -0
- package/src/windowing/count-sliding/index.js +6 -0
- package/src/windowing/count-sliding/is-not-full.js +5 -0
- package/src/windowing/count-sliding/push.js +15 -0
- package/src/windowing/count-sliding/reset.js +11 -0
- package/src/windowing/count-sliding/size.js +5 -0
- package/src/windowing/time-sliding/create.js +25 -0
- package/src/windowing/time-sliding/push.js +42 -0
- package/CODE_OF_CONDUCT.md +0 -46
- package/CONTRIBUTING.md +0 -132
- package/docs/.nojekyll +0 -0
- package/docs/CONTRIBUTING.md +0 -134
- package/docs/README.md +0 -132
- package/docs/_coverpage.md +0 -26
- package/docs/_media/bg.png +0 -0
- package/docs/_media/wink-logo.png +0 -0
- package/docs/_navbar.md +0 -14
- package/docs/_sidebar.md +0 -2
- package/docs/index.html +0 -44
- package/eslint.config.js +0 -283
|
@@ -0,0 +1,488 @@
|
|
|
1
|
+
// core/storage-manager/questdb/persist-plan.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Build per-insightType persist plans from asset class definition.
|
|
5
|
+
*
|
|
6
|
+
* Pre-compiles column writers at startup for allocation-minimized hot path.
|
|
7
|
+
* Each persist plan is a closure that writes a message to QuestDB via ILP.
|
|
8
|
+
*
|
|
9
|
+
* Integration with persist-if node:
|
|
10
|
+
* - persist-if calls: state.storage.write(insightType, msg, partitionId)
|
|
11
|
+
* - Persist plan signature: persistRow(sender, message, partitionId)
|
|
12
|
+
* (Note: partitionId is internal name, written as 'assetId' column)
|
|
13
|
+
*
|
|
14
|
+
* designatedTimestamp handling:
|
|
15
|
+
* - Validated by semantics loader at startup
|
|
16
|
+
* - Used here to call sender.at() which ends the ILP row
|
|
17
|
+
* - Other timestamp columns use sender.timestampColumn()
|
|
18
|
+
*
|
|
19
|
+
* Two-phase rows — validate, then write (ADR-018: a rejected message
|
|
20
|
+
* costs only itself):
|
|
21
|
+
* - Phase 1 checks every column value against its declared type without touching
|
|
22
|
+
* the sender. Phase 2 opens the ILP row and writes only the values that passed;
|
|
23
|
+
* a failed value costs that column, with a warning naming the expected and
|
|
24
|
+
* received types — never a coercion.
|
|
25
|
+
* - Why the order matters: a value the client rejects mid-row (a number handed to
|
|
26
|
+
* a string column) throws AFTER sender.table() and BEFORE sender.at(), leaving
|
|
27
|
+
* the sender wedged on a half-written row — every later write then fails. One
|
|
28
|
+
* wrong value silently killed 98.6% of a replay's writes in the 2026-06-10
|
|
29
|
+
* silent-write-failure incident. Checking every value before the row opens
|
|
30
|
+
* means the row is never started unless it will complete.
|
|
31
|
+
* For that to hold, phase 1's acceptance checks must match the writers' REAL
|
|
32
|
+
* domain — the client requires integers for int64/timestamp columns and for
|
|
33
|
+
* the designated timestamp, so a merely-finite check would let a fractional
|
|
34
|
+
* value through to a mid-row throw (found and fixed 2026-07-07). The
|
|
35
|
+
* recoverSender() backstop in index.js contains any residual mismatch to
|
|
36
|
+
* one row.
|
|
37
|
+
* - Warnings fire in phase 1 too, before the row opens, so the documented
|
|
38
|
+
* strict mode (an `onWarning` that throws) rejects a bad row with the sender
|
|
39
|
+
* untouched.
|
|
40
|
+
*
|
|
41
|
+
* Async-flush failures — the no-silent-failures contract:
|
|
42
|
+
* - @questdb/nodejs-client v4 declares sender.at() as async; the buffer
|
|
43
|
+
* mutation is sync but the trailing `await this.tryFlush()` can fire a
|
|
44
|
+
* network flush. When that flush fails (HTTP timeout, buffer overflow,
|
|
45
|
+
* QDB unreachable), the rows in that batch are dropped. Per composer's
|
|
46
|
+
* "no silent failures" contract, those drops MUST surface loudly.
|
|
47
|
+
*
|
|
48
|
+
* Failures are routed through the `onDeliveryFailure` callback. When
|
|
49
|
+
* a caller provides one, they own the response (log, retry, alert,
|
|
50
|
+
* stop the flow). When none is provided, the catch handler **throws**
|
|
51
|
+
* the failure so it surfaces as an unhandled rejection and the
|
|
52
|
+
* process crashes loudly — better than running on with silently
|
|
53
|
+
* missing data.
|
|
54
|
+
*
|
|
55
|
+
* This separates two concerns:
|
|
56
|
+
* - `onWarning` — soft, per-row data quality (NaN in float column,
|
|
57
|
+
* null where a value was expected, invalid designated timestamp).
|
|
58
|
+
* These are routine ETL realities; default behaviour stays
|
|
59
|
+
* log-and-skip. One warning fires once per insightType instead
|
|
60
|
+
* of per row: a record field named 'assetId' that differs from
|
|
61
|
+
* the partition id. That mistake sits in the flow's
|
|
62
|
+
* configuration, not in the data, so every row would repeat it.
|
|
63
|
+
* The field is ignored either way; the column stores the
|
|
64
|
+
* partition id.
|
|
65
|
+
* - `onDeliveryFailure` — hard, batch-level data loss. Default
|
|
66
|
+
* behaviour: throw, so production cannot run with silent drops.
|
|
67
|
+
*
|
|
68
|
+
* `err.code` (setup-time throws per ADR-018 fail-fast setup):
|
|
69
|
+
* - `INVALID_CONFIG` — buildPersistPlans called with non-function
|
|
70
|
+
* `onWarning` or non-function `onDeliveryFailure`; or a table/column name
|
|
71
|
+
* the client's own ILP rules reject (checked at plan build by driving each
|
|
72
|
+
* name through a throwaway client buffer — a name is not a value, so the
|
|
73
|
+
* per-message phase 1 below cannot catch it).
|
|
74
|
+
*
|
|
75
|
+
* `err.code` (runtime throws / unhandled rejections):
|
|
76
|
+
* - `DELIVERY_FAILED` — sender.at() async flush rejected and no
|
|
77
|
+
* `onDeliveryFailure` callback was provided.
|
|
78
|
+
*
|
|
79
|
+
* @see docs/architecture/storage-layer.md
|
|
80
|
+
* @see ADR-018
|
|
81
|
+
*/
|
|
82
|
+
|
|
83
|
+
import { SenderBufferV1 } from '@questdb/nodejs-client';
|
|
84
|
+
|
|
85
|
+
import { QUEST_WRITERS, writeAsString, createFloat64Writer } from './writers.js';
|
|
86
|
+
|
|
87
|
+
// ============================================================================
|
|
88
|
+
// DEFAULT WARNING HANDLER
|
|
89
|
+
// ============================================================================
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Default warning handler for invalid column values.
|
|
93
|
+
* Logs to console in WinkComposer format.
|
|
94
|
+
*
|
|
95
|
+
* Validation behavior:
|
|
96
|
+
* - null/undefined columns: skip column only (QuestDB stores NULL)
|
|
97
|
+
* - NaN/Infinity in numeric columns: skip column only (QuestDB stores NULL)
|
|
98
|
+
* - Wrong-typed column values (e.g. a number where the column is string-typed):
|
|
99
|
+
* skip column only, never coerce — the warning names the expected and
|
|
100
|
+
* received types
|
|
101
|
+
* - Invalid designatedTimestamp: skip entire row
|
|
102
|
+
*
|
|
103
|
+
* For strict mode (throw on any invalid), provide:
|
|
104
|
+
* { onWarning: (msg) => { throw new Error(msg); } }
|
|
105
|
+
*
|
|
106
|
+
* Future extension: return value may control skip-row vs skip-column behavior.
|
|
107
|
+
*
|
|
108
|
+
* @param {string} message - Warning message describing the issue
|
|
109
|
+
*/
|
|
110
|
+
const defaultOnWarning = function ( message ) {
|
|
111
|
+
console.warn( `WinkComposer/questdb: ${message}` );
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Check if column type requires numeric validation (NaN/Infinity check).
|
|
116
|
+
*
|
|
117
|
+
* @param {string} type - Column type from semantics
|
|
118
|
+
* @returns {boolean} true if type should be validated for finite values
|
|
119
|
+
*/
|
|
120
|
+
const isNumericType = ( type ) => type === 'float64' || type === 'int64' || type === 'timestamp';
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Per-type value checks, pre-resolved at plan build time — the same dispatch idea
|
|
124
|
+
* as QUEST_WRITERS. Each check answers one question: may this value be handed to
|
|
125
|
+
* the matching writer? A value that fails is skipped with a warning — never
|
|
126
|
+
* coerced. Coercion would "work" and silently persist garbage (String( 0.79 )
|
|
127
|
+
* into a text column); the warning is what tells the author their glue is wrong.
|
|
128
|
+
*
|
|
129
|
+
* A column type outside this map falls back to the write-as-string writer, which
|
|
130
|
+
* accepts any value, so its check accepts any value too (acceptAny below).
|
|
131
|
+
*
|
|
132
|
+
* @type {Object.<string, function(*): boolean>}
|
|
133
|
+
*/
|
|
134
|
+
const ACCEPTS = Object.create( null );
|
|
135
|
+
ACCEPTS.float64 = ( value ) => Number.isFinite( value );
|
|
136
|
+
// The client's integer writers REQUIRE integers — probe-verified against
|
|
137
|
+
// 4.2.0: intColumn( 'c', 1.5 ) throws "Value must be an integer" and
|
|
138
|
+
// timestampColumn( 'c', 1.5, 'ms' ) throws "Timestamp value must be an
|
|
139
|
+
// integer or BigInt". A finite non-integer passing phase 1 would throw
|
|
140
|
+
// mid-row in phase 2 — the exact wedge this file exists to prevent.
|
|
141
|
+
// bigint is accepted: the int64/timestamp writers document
|
|
142
|
+
// `number|bigint` and convert before the client call.
|
|
143
|
+
ACCEPTS.int64 = ( value ) => Number.isInteger( value ) || typeof value === 'bigint';
|
|
144
|
+
ACCEPTS.timestamp = ACCEPTS.int64;
|
|
145
|
+
ACCEPTS.string = ( value ) => typeof value === 'string';
|
|
146
|
+
ACCEPTS.bool = ( value ) => typeof value === 'boolean';
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Acceptance check for column types without a dedicated writer: the
|
|
150
|
+
* write-as-string fallback stringifies anything, so any value is writable.
|
|
151
|
+
*
|
|
152
|
+
* @returns {boolean} always true
|
|
153
|
+
*/
|
|
154
|
+
const acceptAny = () => true;
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Builds the reason text for a value that failed its phase-1 acceptance
|
|
158
|
+
* check. Called only on the rare skip path, so its allocations are
|
|
159
|
+
* acceptable (same budget as the error returns in index.js).
|
|
160
|
+
*
|
|
161
|
+
* @param {*} rawValue - the rejected value
|
|
162
|
+
* @param {boolean} isNumeric - whether the column type is numeric
|
|
163
|
+
* @param {string} expectedType - declared column type, named in the message
|
|
164
|
+
* @returns {string} plain reason text for the skip warning
|
|
165
|
+
*/
|
|
166
|
+
const skipReason = function ( rawValue, isNumeric, expectedType ) {
|
|
167
|
+
if ( rawValue === null ) return 'null';
|
|
168
|
+
if ( rawValue === undefined ) return 'undefined';
|
|
169
|
+
if ( isNumeric && typeof rawValue === 'number' ) {
|
|
170
|
+
if ( Number.isNaN( rawValue ) ) return 'NaN';
|
|
171
|
+
// A finite number can only be rejected by the integer-required
|
|
172
|
+
// types (int64/timestamp/designated timestamp) — float64 accepts
|
|
173
|
+
// every finite number.
|
|
174
|
+
return Number.isFinite( rawValue ) ? 'non-integer' : 'non-finite';
|
|
175
|
+
}
|
|
176
|
+
return `wrong-typed (expected ${expectedType}, received ${typeof rawValue})`;
|
|
177
|
+
}; // skipReason()
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Asserts every ILP name a plan will write — the table name and each column
|
|
181
|
+
* name — by driving them once through a throwaway client buffer, so the
|
|
182
|
+
* client's OWN name rules do the checking. No mirrored rules that could
|
|
183
|
+
* drift from the client; the validator is the client.
|
|
184
|
+
*
|
|
185
|
+
* Why here, at plan build: phase 1 validates VALUES per message, but a name
|
|
186
|
+
* is not a value — a bad name would surface as a mid-row client throw at
|
|
187
|
+
* write time, the very failure mode the two-phase split exists to prevent.
|
|
188
|
+
* Names come from the asset class, which is fully known at startup, so a bad
|
|
189
|
+
* one fails the deployment here with a classified INVALID_CONFIG instead of
|
|
190
|
+
* failing the stream later.
|
|
191
|
+
*
|
|
192
|
+
* The scratch buffer is reset before each name, so nothing accumulates and
|
|
193
|
+
* a long column list can never trip a buffer limit.
|
|
194
|
+
*
|
|
195
|
+
* @param {string} tableName - Fully built table name ({prefix}_{insightType})
|
|
196
|
+
* @param {string[]} columnNames - Column names the plan will write
|
|
197
|
+
* @throws {Error} classified `INVALID_CONFIG` naming the offending name
|
|
198
|
+
*/
|
|
199
|
+
const assertIlpNames = function ( tableName, columnNames ) {
|
|
200
|
+
// Client defaults suffice: name validation depends only on the client's
|
|
201
|
+
// default max name length (127), matching the sender the plans run on.
|
|
202
|
+
const scratch = new SenderBufferV1( {} );
|
|
203
|
+
let kind = 'table name';
|
|
204
|
+
let current = tableName;
|
|
205
|
+
try {
|
|
206
|
+
scratch.table( tableName );
|
|
207
|
+
kind = 'column name';
|
|
208
|
+
for ( let i = 0; i < columnNames.length; i += 1 ) {
|
|
209
|
+
current = columnNames[ i ];
|
|
210
|
+
scratch.reset();
|
|
211
|
+
scratch.table( 't' );
|
|
212
|
+
scratch.symbol( current, 'x' );
|
|
213
|
+
}
|
|
214
|
+
} catch ( err ) {
|
|
215
|
+
const wrapped = new Error(
|
|
216
|
+
`WinkComposer/questdb: invalid ILP ${kind} '${current}' — ${err.message}`
|
|
217
|
+
);
|
|
218
|
+
wrapped.code = 'INVALID_CONFIG';
|
|
219
|
+
wrapped.cause = err;
|
|
220
|
+
throw wrapped;
|
|
221
|
+
}
|
|
222
|
+
}; // assertIlpNames()
|
|
223
|
+
|
|
224
|
+
// ============================================================================
|
|
225
|
+
// PERSIST PLAN BUILDER
|
|
226
|
+
// ============================================================================
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Build per-insightType persist plans from asset class definition.
|
|
230
|
+
*
|
|
231
|
+
* Each plan is a closure that works in two phases — validate, then write:
|
|
232
|
+
* 1. Phase 1 validates the designatedTimestamp (skips the row if invalid) and
|
|
233
|
+
* checks every column value against its declared type. No sender calls.
|
|
234
|
+
* 2. Phase 2 sets the table name `{tablePrefix}_{insightType}`, writes assetId
|
|
235
|
+
* as SYMBOL, writes each column that passed via its pre-resolved type writer
|
|
236
|
+
* (a failed column is skipped with a warning), and ends the row with
|
|
237
|
+
* sender.at().
|
|
238
|
+
*
|
|
239
|
+
* Validation behavior (fault-tolerant, never throws):
|
|
240
|
+
* - Invalid designatedTimestamp (null/undefined/NaN/Infinity): warn + skip entire row
|
|
241
|
+
* - null/undefined column: warn + skip column (QuestDB stores NULL)
|
|
242
|
+
* - NaN/Infinity in numeric column: warn + skip column (QuestDB stores NULL).
|
|
243
|
+
* This is the designed end of composer's NaN propagation: an upstream node
|
|
244
|
+
* marks an invalid input by publishing NaN, and the persist plan lands it as
|
|
245
|
+
* a NULL column while the rest of the row survives.
|
|
246
|
+
* - Wrong-typed value in any column: warn + skip column, never coerce
|
|
247
|
+
*
|
|
248
|
+
* For strict mode, provide onWarning that throws.
|
|
249
|
+
*
|
|
250
|
+
* @param {Object} assetClass - Asset class definition with columns and insightTypes
|
|
251
|
+
* @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
|
|
252
|
+
* @param {Object} [options] - Optional configuration
|
|
253
|
+
* @param {function} [options.onWarning] - Soft per-row warning callback for
|
|
254
|
+
* invalid values (NaN, null, invalid timestamp). Default: console.warn.
|
|
255
|
+
* @param {function} [options.onDeliveryFailure] - Hard batch-level failure
|
|
256
|
+
* callback for `sender.at()` async flush rejection (HTTP timeout, buffer
|
|
257
|
+
* overflow, QDB unreachable). Default: throw — surfaces as an unhandled
|
|
258
|
+
* rejection so production cannot run with silent row drops.
|
|
259
|
+
* @returns {Object.<string, function>} Map of insightType to persistRow function
|
|
260
|
+
*/
|
|
261
|
+
const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
|
|
262
|
+
const providedOnWarning = options && options.onWarning;
|
|
263
|
+
const providedOnDeliveryFailure = options && options.onDeliveryFailure;
|
|
264
|
+
|
|
265
|
+
if ( providedOnWarning !== undefined && typeof providedOnWarning !== 'function' ) {
|
|
266
|
+
// Per ADR-018, setup-time throws carry classified err.code.
|
|
267
|
+
const err = new Error( 'WinkComposer/questdb: onWarning must be a function' );
|
|
268
|
+
err.code = 'INVALID_CONFIG';
|
|
269
|
+
throw err;
|
|
270
|
+
}
|
|
271
|
+
if ( providedOnDeliveryFailure !== undefined && typeof providedOnDeliveryFailure !== 'function' ) {
|
|
272
|
+
const err = new Error( 'WinkComposer/questdb: onDeliveryFailure must be a function' );
|
|
273
|
+
err.code = 'INVALID_CONFIG';
|
|
274
|
+
throw err;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
const onWarning = providedOnWarning || defaultOnWarning;
|
|
278
|
+
const onDeliveryFailure = providedOnDeliveryFailure || null;
|
|
279
|
+
const plansByInsightType = Object.create( null );
|
|
280
|
+
const insightTypes = assetClass.insightTypes || Object.create( null );
|
|
281
|
+
const columns = assetClass.columns;
|
|
282
|
+
|
|
283
|
+
const insightTypeNames = Object.keys( insightTypes );
|
|
284
|
+
|
|
285
|
+
for ( let i = 0; i < insightTypeNames.length; i += 1 ) {
|
|
286
|
+
const insightTypeName = insightTypeNames[ i ];
|
|
287
|
+
const insightTypeSpec = insightTypes[ insightTypeName ];
|
|
288
|
+
const persistedColumnNames = insightTypeSpec.columns;
|
|
289
|
+
const designatedTimestamp = insightTypeSpec.designatedTimestamp;
|
|
290
|
+
|
|
291
|
+
// The column name 'assetId' is reserved. Composer writes that
|
|
292
|
+
// column itself, from the partition id. A persisted column with
|
|
293
|
+
// the same name would make the CREATE TABLE ask for 'assetId'
|
|
294
|
+
// twice. That happens whether the name appears in the columns
|
|
295
|
+
// list or as the designated timestamp. Fail fast here and name
|
|
296
|
+
// the fix; otherwise QuestDB answers at table creation with its
|
|
297
|
+
// raw "Duplicate column" error. A dictionary column named
|
|
298
|
+
// 'assetId' that no insightType persists stays legal.
|
|
299
|
+
if ( persistedColumnNames.includes( 'assetId' ) || designatedTimestamp === 'assetId' ) {
|
|
300
|
+
const err = new Error(
|
|
301
|
+
`WinkComposer/questdb: insightType '${insightTypeName}' uses reserved column name 'assetId' — ` +
|
|
302
|
+
'composer writes this column automatically from the flow\'s .assetId() field (the partition id); ' +
|
|
303
|
+
'rename the semantics column or remove it from this insightType'
|
|
304
|
+
);
|
|
305
|
+
err.code = 'INVALID_CONFIG';
|
|
306
|
+
throw err;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// Pre-compile writer steps
|
|
310
|
+
// designatedTimestamp is handled separately via sender.at()
|
|
311
|
+
// Other columns use their type-specific writers
|
|
312
|
+
const stepNames = [];
|
|
313
|
+
const stepWriters = [];
|
|
314
|
+
const stepIsNumeric = [];
|
|
315
|
+
const stepTypes = [];
|
|
316
|
+
const stepAccepts = [];
|
|
317
|
+
|
|
318
|
+
for ( let j = 0; j < persistedColumnNames.length; j += 1 ) {
|
|
319
|
+
const columnName = persistedColumnNames[ j ];
|
|
320
|
+
|
|
321
|
+
// Skip designatedTimestamp - handled separately via sender.at()
|
|
322
|
+
if ( columnName !== designatedTimestamp ) {
|
|
323
|
+
const columnSpec = columns[ columnName ];
|
|
324
|
+
const columnType = columnSpec ? columnSpec.type : 'string';
|
|
325
|
+
|
|
326
|
+
stepNames.push( columnName );
|
|
327
|
+
stepIsNumeric.push( isNumericType( columnType ) );
|
|
328
|
+
stepTypes.push( columnType );
|
|
329
|
+
stepAccepts.push( ACCEPTS[ columnType ] || acceptAny );
|
|
330
|
+
|
|
331
|
+
// float64 columns use resolution-aware writer factory
|
|
332
|
+
// Note: columnSpec is guaranteed to exist here since columnType='float64'
|
|
333
|
+
// requires columnSpec.type to be 'float64' (line 65)
|
|
334
|
+
if ( columnType === 'float64' ) {
|
|
335
|
+
stepWriters.push( createFloat64Writer( columnSpec.resolution ) );
|
|
336
|
+
} else {
|
|
337
|
+
stepWriters.push( QUEST_WRITERS[ columnType ] || writeAsString );
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const stepCount = stepNames.length;
|
|
343
|
+
const tableName = tablePrefix + '_' + insightTypeName;
|
|
344
|
+
// Fail-fast at startup: a bad table or column name would otherwise
|
|
345
|
+
// throw inside the client mid-row at write time.
|
|
346
|
+
assertIlpNames( tableName, stepNames );
|
|
347
|
+
// Scratch array reused on every row: phase 1 records which columns passed
|
|
348
|
+
// validation, phase 2 reads it. Allocated once here, never per message.
|
|
349
|
+
const stepValueOk = new Array( stepCount );
|
|
350
|
+
|
|
351
|
+
// Once-flag for the assetId-mismatch warning below, allocated once
|
|
352
|
+
// per insightType (never per row). The mismatch is a mistake in
|
|
353
|
+
// the flow's configuration, so it would repeat on every row.
|
|
354
|
+
// One report carries all the information.
|
|
355
|
+
let assetIdMismatchWarned = false;
|
|
356
|
+
|
|
357
|
+
// Create closure that captures pre-resolved references
|
|
358
|
+
// Interface: persistRow(sender, message, partitionId) -> boolean
|
|
359
|
+
// Returns true when a row was opened and completed on the sender,
|
|
360
|
+
// false when phase 1 skipped the whole row (the sender was never
|
|
361
|
+
// touched). The caller's buffered-row accounting keys off this —
|
|
362
|
+
// a skipped row must not count as buffered.
|
|
363
|
+
// Note: partitionId is internal name, written as 'assetId' column to QuestDB
|
|
364
|
+
plansByInsightType[ insightTypeName ] = function ( sender, message, partitionId ) {
|
|
365
|
+
// ---- Phase 1: validate. No sender calls — nothing irreversible
|
|
366
|
+
// happens until every value has been checked (see file header).
|
|
367
|
+
|
|
368
|
+
// designatedTimestamp first - if invalid, skip entire row
|
|
369
|
+
const tsValue = message[ designatedTimestamp ];
|
|
370
|
+
if ( tsValue === undefined || tsValue === null ) {
|
|
371
|
+
onWarning(
|
|
372
|
+
`designatedTimestamp '${designatedTimestamp}' is ${tsValue === null ? 'null' : 'undefined'} ` +
|
|
373
|
+
`in insightType '${insightTypeName}' (asset: ${partitionId}) - row skipped`
|
|
374
|
+
);
|
|
375
|
+
return false;
|
|
376
|
+
}
|
|
377
|
+
// Integer-or-bigint, matching the client's own .at() validation
|
|
378
|
+
// (probe-verified: at( ...000.5, 'ms' ) throws "Designated
|
|
379
|
+
// timestamp must be an integer or BigInt" — and it throws AFTER
|
|
380
|
+
// the whole row is written, so catching it here is the only
|
|
381
|
+
// place the row survives intact).
|
|
382
|
+
if ( !Number.isInteger( tsValue ) && typeof tsValue !== 'bigint' ) {
|
|
383
|
+
onWarning(
|
|
384
|
+
`designatedTimestamp '${designatedTimestamp}' is ${skipReason( tsValue, true, 'timestamp' )} ` +
|
|
385
|
+
`in insightType '${insightTypeName}' (asset: ${partitionId}) - row skipped`
|
|
386
|
+
);
|
|
387
|
+
return false;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// A record field named 'assetId' never sets the assetId column.
|
|
391
|
+
// That column always stores the partition id. A record value
|
|
392
|
+
// that differs is almost always one mistake: trying to relabel
|
|
393
|
+
// identity in the record. It is reported once per insightType,
|
|
394
|
+
// never silently dropped (ADR-018 no-silent-failures). The
|
|
395
|
+
// flag advances only after onWarning returns. So under strict
|
|
396
|
+
// mode, where onWarning throws, every mismatched row is
|
|
397
|
+
// rejected here with the sender untouched.
|
|
398
|
+
if ( !assetIdMismatchWarned && message.assetId !== undefined && message.assetId !== partitionId ) {
|
|
399
|
+
onWarning(
|
|
400
|
+
`record field 'assetId' (${message.assetId}) ignored in insightType '${insightTypeName}' ` +
|
|
401
|
+
`(asset: ${partitionId}) — the assetId column always stores the partition id; to change ` +
|
|
402
|
+
'stored identity, change the flow\'s .assetId() field; reported once per insightType'
|
|
403
|
+
);
|
|
404
|
+
assetIdMismatchWarned = true;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// Check every column value against its declared type. A failed
|
|
408
|
+
// check marks the column for a silent skip in phase 2 (QuestDB
|
|
409
|
+
// stores NULL) — the same treatment null already gets — and
|
|
410
|
+
// warns HERE, before the row opens, so an onWarning that throws
|
|
411
|
+
// (documented strict mode) rejects the row with the sender
|
|
412
|
+
// untouched instead of wedging it mid-row. Composer's NaN
|
|
413
|
+
// propagation ends here exactly as before: a NaN in a numeric
|
|
414
|
+
// column fails its acceptance check and lands as a NULL column
|
|
415
|
+
// while the row survives.
|
|
416
|
+
for ( let k = 0; k < stepCount; k += 1 ) {
|
|
417
|
+
const rawValue = message[ stepNames[ k ] ];
|
|
418
|
+
stepValueOk[ k ] = ( rawValue !== null ) && ( rawValue !== undefined ) && stepAccepts[ k ]( rawValue );
|
|
419
|
+
if ( !stepValueOk[ k ] ) {
|
|
420
|
+
onWarning(
|
|
421
|
+
`column '${stepNames[ k ]}' is ${skipReason( rawValue, stepIsNumeric[ k ], stepTypes[ k ] )} ` +
|
|
422
|
+
`in insightType '${insightTypeName}' (asset: ${partitionId}) — column skipped`
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// ---- Phase 2: write. The row opens only after every value passed,
|
|
428
|
+
// so it is never left half-written by a rejected value.
|
|
429
|
+
sender.table( tableName );
|
|
430
|
+
sender.symbol( 'assetId', partitionId );
|
|
431
|
+
|
|
432
|
+
for ( let k = 0; k < stepCount; k += 1 ) {
|
|
433
|
+
if ( stepValueOk[ k ] ) {
|
|
434
|
+
stepWriters[ k ]( sender, stepNames[ k ], message[ stepNames[ k ] ] );
|
|
435
|
+
}
|
|
436
|
+
// A skipped column (QuestDB stores NULL) was already warned
|
|
437
|
+
// about in phase 1 — before the row opened.
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// Designated timestamp (ends the row in ILP).
|
|
441
|
+
//
|
|
442
|
+
// Async flush failure handling — the no-silent-failures contract:
|
|
443
|
+
//
|
|
444
|
+
// sender.at() is async — the buffer mutation is sync but the
|
|
445
|
+
// trailing `await this.tryFlush()` may fire a network flush.
|
|
446
|
+
// When that flush fails (HTTP timeout, buffer overflow at the
|
|
447
|
+
// client's max_buf_size, QDB unreachable), the rows in that
|
|
448
|
+
// batch are dropped. Composer must NOT silently lose data.
|
|
449
|
+
//
|
|
450
|
+
// Routing rules:
|
|
451
|
+
// - If a caller provided `onDeliveryFailure`, route the
|
|
452
|
+
// failure to it. The caller owns the response (log,
|
|
453
|
+
// retry-budget, alert, stop the flow).
|
|
454
|
+
// - Otherwise, throw a classified `DELIVERY_FAILED` error.
|
|
455
|
+
// Inside a .catch() handler this surfaces as an
|
|
456
|
+
// unhandled rejection — Node logs it loudly and (15+)
|
|
457
|
+
// terminates the process. Loud failure beats silent
|
|
458
|
+
// loss every time.
|
|
459
|
+
const atResult = sender.at( tsValue, 'ms' );
|
|
460
|
+
if ( atResult && typeof atResult.catch === 'function' ) {
|
|
461
|
+
atResult.catch( ( err ) => {
|
|
462
|
+
if ( onDeliveryFailure ) {
|
|
463
|
+
onDeliveryFailure( err, { tableName } );
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
const failure = new Error(
|
|
467
|
+
`WinkComposer/questdb: silent data loss — sender.at() async flush failed for table '${tableName}': ${err.message}. ` +
|
|
468
|
+
'Rows in this batch were dropped. Per the no-silent-failures contract this is a hard failure. ' +
|
|
469
|
+
'Provide an `onDeliveryFailure` callback in the storage config to handle these explicitly.'
|
|
470
|
+
);
|
|
471
|
+
failure.code = 'DELIVERY_FAILED';
|
|
472
|
+
failure.cause = err;
|
|
473
|
+
throw failure;
|
|
474
|
+
} );
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
return true;
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
return plansByInsightType;
|
|
482
|
+
};
|
|
483
|
+
|
|
484
|
+
// ============================================================================
|
|
485
|
+
// EXPORTS
|
|
486
|
+
// ============================================================================
|
|
487
|
+
|
|
488
|
+
export { buildPersistPlans, defaultOnWarning };
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// core/storage-manager/questdb/writers.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview QuestDB column writers mapped to semantics COLUMN_TYPES.
|
|
5
|
+
*
|
|
6
|
+
* Each writer is a function that calls the appropriate QuestDB Sender method
|
|
7
|
+
* for the given column type. These are pre-resolved at startup and stored
|
|
8
|
+
* in persist plans for allocation-minimized hot path execution.
|
|
9
|
+
*
|
|
10
|
+
* Timestamp handling:
|
|
11
|
+
* - rowTimestamp (designated): handled via sender.at() which ends the row
|
|
12
|
+
* - Additional timestamps: use QUEST_WRITERS.timestamp via sender.timestampColumn()
|
|
13
|
+
*
|
|
14
|
+
* Resolution-aware float64 quantization is delegated to the shared
|
|
15
|
+
* `core/utils/quantize` helper so QDB and the terminal emitter (and any
|
|
16
|
+
* future adapter that honours declared resolution) compute identical
|
|
17
|
+
* rounded values for the same input. Cross-sink alignment by construction.
|
|
18
|
+
*
|
|
19
|
+
* @see https://questdb.com/docs/clients/ingest-node/
|
|
20
|
+
* @see ../../utils/quantize/index.js (shared quantizer used by createFloat64Writer)
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { buildResolutionQuantizer } from '../../utils/quantize/index.js';
|
|
24
|
+
|
|
25
|
+
// ============================================================================
|
|
26
|
+
// TYPE WRITERS
|
|
27
|
+
// ============================================================================
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Type-specific column writers for QuestDB ILP.
|
|
31
|
+
* Keys match semantics COLUMN_TYPES from column-schema.js.
|
|
32
|
+
*
|
|
33
|
+
* @type {Object.<string, function(Object, string, *): void>}
|
|
34
|
+
*/
|
|
35
|
+
const QUEST_WRITERS = Object.create( null );
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Write float64 value.
|
|
39
|
+
*
|
|
40
|
+
* @param {Object} sender - QuestDB Sender instance
|
|
41
|
+
* @param {string} columnName - Column name
|
|
42
|
+
* @param {number} value - Float value
|
|
43
|
+
*/
|
|
44
|
+
QUEST_WRITERS.float64 = function ( sender, columnName, value ) {
|
|
45
|
+
sender.floatColumn( columnName, value );
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Factory to create a resolution-aware float64 writer.
|
|
50
|
+
*
|
|
51
|
+
* Wraps the shared `buildResolutionQuantizer` (in `core/utils/quantize`)
|
|
52
|
+
* with QDB's sender-calling glue. The shared quantizer pre-computes the
|
|
53
|
+
* inverse resolution and decimal-place count once at factory time, so
|
|
54
|
+
* the per-message hot path is one quantize call + one `sender.floatColumn`
|
|
55
|
+
* call — no allocations, no recomputation.
|
|
56
|
+
*
|
|
57
|
+
* When the shared helper returns `null` (resolution undefined or 1),
|
|
58
|
+
* we route to the default `QUEST_WRITERS.float64` passthrough writer
|
|
59
|
+
* so callers do not pay the wrapper cost for the common case of "no
|
|
60
|
+
* quantization needed".
|
|
61
|
+
*
|
|
62
|
+
* @param {number} resolution - Resolution from semantics (e.g., 0.1, 0.01)
|
|
63
|
+
* @returns {function(Object, string, number): void} Writer function
|
|
64
|
+
*/
|
|
65
|
+
const createFloat64Writer = function ( resolution ) {
|
|
66
|
+
const quantize = buildResolutionQuantizer( resolution );
|
|
67
|
+
if ( !quantize ) return QUEST_WRITERS.float64;
|
|
68
|
+
|
|
69
|
+
return function ( sender, columnName, value ) {
|
|
70
|
+
sender.floatColumn( columnName, quantize( value ) );
|
|
71
|
+
};
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Write int64 value.
|
|
76
|
+
*
|
|
77
|
+
* WARNING: JavaScript numbers lose precision above Number.MAX_SAFE_INTEGER (2^53 - 1).
|
|
78
|
+
* BigInt is converted to Number here, which may cause precision loss for very large values.
|
|
79
|
+
*
|
|
80
|
+
* @param {Object} sender - QuestDB Sender instance
|
|
81
|
+
* @param {string} columnName - Column name
|
|
82
|
+
* @param {number|bigint} value - Integer value
|
|
83
|
+
*/
|
|
84
|
+
QUEST_WRITERS.int64 = function ( sender, columnName, value ) {
|
|
85
|
+
sender.intColumn( columnName, typeof value === 'bigint' ? Number( value ) : value );
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Write boolean value.
|
|
90
|
+
*
|
|
91
|
+
* @param {Object} sender - QuestDB Sender instance
|
|
92
|
+
* @param {string} columnName - Column name
|
|
93
|
+
* @param {boolean} value - Boolean value
|
|
94
|
+
*/
|
|
95
|
+
QUEST_WRITERS.bool = function ( sender, columnName, value ) {
|
|
96
|
+
sender.booleanColumn( columnName, value );
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Write string value.
|
|
101
|
+
*
|
|
102
|
+
* @param {Object} sender - QuestDB Sender instance
|
|
103
|
+
* @param {string} columnName - Column name
|
|
104
|
+
* @param {string} value - String value
|
|
105
|
+
*/
|
|
106
|
+
QUEST_WRITERS.string = function ( sender, columnName, value ) {
|
|
107
|
+
sender.stringColumn( columnName, value );
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Write timestamp value (for additional timestamp columns, NOT rowTimestamp).
|
|
112
|
+
* The designated rowTimestamp is handled via sender.at() which ends the row.
|
|
113
|
+
*
|
|
114
|
+
* @param {Object} sender - QuestDB Sender instance
|
|
115
|
+
* @param {string} columnName - Column name
|
|
116
|
+
* @param {number} value - Timestamp in milliseconds since epoch
|
|
117
|
+
*/
|
|
118
|
+
QUEST_WRITERS.timestamp = function ( sender, columnName, value ) {
|
|
119
|
+
sender.timestampColumn( columnName, value, 'ms' );
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
// ============================================================================
|
|
123
|
+
// FALLBACK WRITER
|
|
124
|
+
// ============================================================================
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Fallback writer that converts value to string.
|
|
128
|
+
* Used for unknown column types or when explicit string conversion is needed.
|
|
129
|
+
*
|
|
130
|
+
* @param {Object} sender - QuestDB Sender instance
|
|
131
|
+
* @param {string} columnName - Column name
|
|
132
|
+
* @param {*} value - Any value (converted to string)
|
|
133
|
+
*/
|
|
134
|
+
const writeAsString = function ( sender, columnName, value ) {
|
|
135
|
+
sender.stringColumn( columnName, String( value ) );
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
// ============================================================================
|
|
139
|
+
// EXPORTS
|
|
140
|
+
// ============================================================================
|
|
141
|
+
|
|
142
|
+
export { QUEST_WRITERS, writeAsString, createFloat64Writer };
|