@winkjs/composer 0.0.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +106 -80
- package/package.json +56 -28
- package/src/composer.js +75 -0
- package/src/core/batch/aoo-encoder.js +148 -0
- package/src/core/batch/index.js +12 -0
- package/src/core/batch/sliding-window.js +91 -0
- package/src/core/codec/index.js +137 -0
- package/src/core/emitter-manager/mqtt/constants.js +102 -0
- package/src/core/emitter-manager/mqtt/emitter.js +780 -0
- package/src/core/emitter-manager/mqtt/index.js +190 -0
- package/src/core/emitter-manager/mqtt/mqtt-store.js +538 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +98 -0
- package/src/core/emitter-manager/terminal/formatters.js +160 -0
- package/src/core/emitter-manager/terminal/index.js +341 -0
- package/src/core/env-vars.js +245 -0
- package/src/core/mqtt-protocol.js +47 -0
- package/src/core/partition-manager/index.js +7 -0
- package/src/core/partition-manager/init.js +44 -0
- package/src/core/partition-manager/introspect.js +15 -0
- package/src/core/partition-manager/publish-to.js +6 -0
- package/src/core/partition-manager/recompute.js +5 -0
- package/src/core/partition-manager/reset.js +5 -0
- package/src/core/partition-manager/update.js +264 -0
- package/src/core/partition-manager/validate-flow.js +157 -0
- package/src/core/semantics/digest.js +128 -0
- package/src/core/semantics/index.js +83 -0
- package/src/core/semantics/loader.js +571 -0
- package/src/core/semantics/schemas/asset-class-schema.js +218 -0
- package/src/core/semantics/schemas/column-schema.js +651 -0
- package/src/core/semantics/schemas/enum-schema.js +153 -0
- package/src/core/semantics/schemas/index.js +51 -0
- package/src/core/semantics/warnings.js +141 -0
- package/src/core/shutdown-manager/index.js +289 -0
- package/src/core/source-manager/csv/index.js +171 -0
- package/src/core/source-manager/csv/start.js +456 -0
- package/src/core/source-manager/mqtt/client.js +406 -0
- package/src/core/source-manager/mqtt/constants.js +147 -0
- package/src/core/source-manager/mqtt/dedup.js +244 -0
- package/src/core/source-manager/mqtt/index.js +260 -0
- package/src/core/source-manager/mqtt/status.js +546 -0
- package/src/core/source-manager/test-harness/comparator.js +226 -0
- package/src/core/source-manager/test-harness/field-generator.js +85 -0
- package/src/core/source-manager/test-harness/fuzz.js +87 -0
- package/src/core/source-manager/test-harness/index.js +115 -0
- package/src/core/source-manager/test-harness/prng.js +57 -0
- package/src/core/source-manager/test-harness/start.js +232 -0
- package/src/core/source-manager/test-harness/validate.js +213 -0
- package/src/core/storage-manager/questdb/assert-columns.js +120 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +187 -0
- package/src/core/storage-manager/questdb/index.js +1251 -0
- package/src/core/storage-manager/questdb/persist-plan.js +488 -0
- package/src/core/storage-manager/questdb/writers.js +142 -0
- package/src/core/test-utils/sealed-record.js +80 -0
- package/src/core/test-utils/soak-signature.js +149 -0
- package/src/core/test-utils/tcp-proxy.js +106 -0
- package/src/core/tunable/helpers.js +254 -0
- package/src/core/tunable/index.js +87 -0
- package/src/core/utils/console-formatter/index.js +207 -0
- package/src/core/utils/create-smart-store-as/index.js +98 -0
- package/src/core/utils/create-smart-store-as/sanitize-field.js +46 -0
- package/src/core/utils/create-smart-store-as/smart-truncate.js +93 -0
- package/src/core/utils/flow/index.js +5 -0
- package/src/core/utils/flow/make-collision-checker.js +23 -0
- package/src/core/utils/flow/to-kebab.js +20 -0
- package/src/core/utils/half-life/index.js +111 -0
- package/src/core/utils/node/disable.js +11 -0
- package/src/core/utils/node/enable.js +11 -0
- package/src/core/utils/node/execute-triggers.js +40 -0
- package/src/core/utils/node/index.js +10 -0
- package/src/core/utils/node/pause.js +11 -0
- package/src/core/utils/node/populate-predicate-inputs.js +14 -0
- package/src/core/utils/node/publish-nan.js +19 -0
- package/src/core/utils/node/reset-predicate-inputs.js +11 -0
- package/src/core/utils/node/unpause.js +11 -0
- package/src/core/utils/node/validate-spec.js +22 -0
- package/src/core/utils/options/resolve-field-keyed.js +115 -0
- package/src/core/utils/quantize/index.js +75 -0
- package/src/core/utils/stats/formulas.js +67 -0
- package/src/core/utils/stats/index.js +3 -0
- package/src/core/utils/validate/composer-validators.js +141 -0
- package/src/core/utils/validate/helpers.js +206 -0
- package/src/core/utils/validate/index.js +12 -0
- package/src/core/utils/validate/type-validators.js +538 -0
- package/src/core/utils/validate/validate.js +107 -0
- package/src/core/utils/validate/validators.js +177 -0
- package/src/core/wiring/assert-handle.js +57 -0
- package/src/core/wiring/assert-module.js +63 -0
- package/src/core/wiring/index.js +5 -0
- package/src/core/wiring/node-type-to-module.js +17 -0
- package/src/core/wiring/wire-emitters.js +256 -0
- package/src/core/wiring/wire-linear-graph.js +25 -0
- package/src/core/wiring/wire-node.js +157 -0
- package/src/core/wiring/wire-semantics.js +173 -0
- package/src/core/wiring/wire-storages.js +344 -0
- package/src/flow/check-output-collisions.js +93 -0
- package/src/flow/consts.js +9 -0
- package/src/flow/deep-clone-spec.js +64 -0
- package/src/flow/driver.js +171 -0
- package/src/flow/expand-single-input-array.js +76 -0
- package/src/flow/expand-template-spec.js +93 -0
- package/src/flow/flow-api-schemas.js +300 -0
- package/src/flow/flow.js +658 -0
- package/src/flow/for-each.js +225 -0
- package/src/flow/get-signature-pattern.js +31 -0
- package/src/flow/get-target-array.js +46 -0
- package/src/flow/inspect.js +67 -0
- package/src/flow/load-node-module.js +81 -0
- package/src/flow/make-node-method.js +112 -0
- package/src/flow/resolve-pick-by-field.js +77 -0
- package/src/flow/resolve-tunables.js +76 -0
- package/src/flow/run.js +392 -0
- package/src/flow/serialize.js +161 -0
- package/src/flow/signatures-schema.js +222 -0
- package/src/flow/spec-builder.js +65 -0
- package/src/flow/validate-triggers.js +250 -0
- package/src/flow/validate.js +261 -0
- package/src/nodes/accumulate/index.js +21 -0
- package/src/nodes/accumulate/init.js +48 -0
- package/src/nodes/accumulate/introspect.js +123 -0
- package/src/nodes/accumulate/publish-to.js +34 -0
- package/src/nodes/accumulate/recompute.js +27 -0
- package/src/nodes/accumulate/reset.js +25 -0
- package/src/nodes/accumulate/update.js +42 -0
- package/src/nodes/appraise/calibrate.js +64 -0
- package/src/nodes/appraise/decision.js +66 -0
- package/src/nodes/appraise/deviation.js +140 -0
- package/src/nodes/appraise/index.js +23 -0
- package/src/nodes/appraise/init.js +165 -0
- package/src/nodes/appraise/integrate.js +45 -0
- package/src/nodes/appraise/introspect.js +352 -0
- package/src/nodes/appraise/publish-to.js +93 -0
- package/src/nodes/appraise/receptor.js +69 -0
- package/src/nodes/appraise/recompute.js +79 -0
- package/src/nodes/appraise/reset.js +53 -0
- package/src/nodes/appraise/update.js +130 -0
- package/src/nodes/archive/csv/index.js +6 -0
- package/src/nodes/archive/csv/init.js +36 -0
- package/src/nodes/archive/csv/load.js +70 -0
- package/src/nodes/archive/csv/reset.js +14 -0
- package/src/nodes/archive/csv/simple-parser.js +216 -0
- package/src/nodes/archive/csv/test.js +173 -0
- package/src/nodes/archive/csv/update.js +63 -0
- package/src/nodes/butterworth-filter/index.js +18 -0
- package/src/nodes/butterworth-filter/init.js +184 -0
- package/src/nodes/butterworth-filter/introspect.js +228 -0
- package/src/nodes/butterworth-filter/publish-to.js +21 -0
- package/src/nodes/butterworth-filter/recompute.js +13 -0
- package/src/nodes/butterworth-filter/reset.js +30 -0
- package/src/nodes/butterworth-filter/update.js +38 -0
- package/src/nodes/categorize/index.js +19 -0
- package/src/nodes/categorize/init.js +58 -0
- package/src/nodes/categorize/introspect.js +177 -0
- package/src/nodes/categorize/publish-to.js +29 -0
- package/src/nodes/categorize/recompute.js +14 -0
- package/src/nodes/categorize/reset.js +16 -0
- package/src/nodes/categorize/update.js +53 -0
- package/src/nodes/controller/index.js +9 -0
- package/src/nodes/controller/init.js +55 -0
- package/src/nodes/controller/introspect.js +91 -0
- package/src/nodes/controller/publish-to.js +8 -0
- package/src/nodes/controller/recompute.js +9 -0
- package/src/nodes/controller/reset.js +17 -0
- package/src/nodes/controller/update.js +63 -0
- package/src/nodes/diff/index.js +19 -0
- package/src/nodes/diff/init.js +37 -0
- package/src/nodes/diff/introspect.js +132 -0
- package/src/nodes/diff/publish-to.js +21 -0
- package/src/nodes/diff/recompute.js +12 -0
- package/src/nodes/diff/reset.js +12 -0
- package/src/nodes/diff/update.js +37 -0
- package/src/nodes/digest-moments/formulas.js +12 -0
- package/src/nodes/digest-moments/index.js +14 -0
- package/src/nodes/digest-moments/init.js +67 -0
- package/src/nodes/digest-moments/introspect.js +154 -0
- package/src/nodes/digest-moments/publish-to.js +34 -0
- package/src/nodes/digest-moments/recompute.js +14 -0
- package/src/nodes/digest-moments/reset.js +14 -0
- package/src/nodes/digest-moments/update.js +120 -0
- package/src/nodes/dwell-time-tracker/index.js +14 -0
- package/src/nodes/dwell-time-tracker/init.js +59 -0
- package/src/nodes/dwell-time-tracker/introspect.js +137 -0
- package/src/nodes/dwell-time-tracker/publish-to.js +22 -0
- package/src/nodes/dwell-time-tracker/recompute.js +8 -0
- package/src/nodes/dwell-time-tracker/reset.js +23 -0
- package/src/nodes/dwell-time-tracker/update.js +103 -0
- package/src/nodes/emit-if/helpers.js +94 -0
- package/src/nodes/emit-if/index.js +16 -0
- package/src/nodes/emit-if/init.js +67 -0
- package/src/nodes/emit-if/introspect.js +102 -0
- package/src/nodes/emit-if/publish-to.js +13 -0
- package/src/nodes/emit-if/recompute.js +12 -0
- package/src/nodes/emit-if/reset.js +27 -0
- package/src/nodes/emit-if/update.js +93 -0
- package/src/nodes/es-correlation/index.js +19 -0
- package/src/nodes/es-correlation/init.js +73 -0
- package/src/nodes/es-correlation/introspect.js +176 -0
- package/src/nodes/es-correlation/publish-to.js +32 -0
- package/src/nodes/es-correlation/recompute.js +56 -0
- package/src/nodes/es-correlation/reset.js +27 -0
- package/src/nodes/es-correlation/update.js +110 -0
- package/src/nodes/es-mean/index.js +20 -0
- package/src/nodes/es-mean/init.js +58 -0
- package/src/nodes/es-mean/introspect.js +129 -0
- package/src/nodes/es-mean/publish-to.js +19 -0
- package/src/nodes/es-mean/recompute.js +16 -0
- package/src/nodes/es-mean/reset.js +24 -0
- package/src/nodes/es-mean/update.js +69 -0
- package/src/nodes/es-pairwise-correlation/index.js +19 -0
- package/src/nodes/es-pairwise-correlation/init.js +98 -0
- package/src/nodes/es-pairwise-correlation/introspect.js +187 -0
- package/src/nodes/es-pairwise-correlation/publish-to.js +31 -0
- package/src/nodes/es-pairwise-correlation/recompute.js +69 -0
- package/src/nodes/es-pairwise-correlation/reset.js +29 -0
- package/src/nodes/es-pairwise-correlation/update.js +89 -0
- package/src/nodes/es-stats/index.js +21 -0
- package/src/nodes/es-stats/init.js +105 -0
- package/src/nodes/es-stats/introspect.js +157 -0
- package/src/nodes/es-stats/publish-to.js +42 -0
- package/src/nodes/es-stats/recompute.js +74 -0
- package/src/nodes/es-stats/reset.js +49 -0
- package/src/nodes/es-stats/update-envelope.js +44 -0
- package/src/nodes/es-stats/update-welford.js +64 -0
- package/src/nodes/es-stats/update.js +59 -0
- package/src/nodes/index.js +83 -0
- package/src/nodes/invert-flag/index.js +20 -0
- package/src/nodes/invert-flag/init.js +36 -0
- package/src/nodes/invert-flag/introspect.js +115 -0
- package/src/nodes/invert-flag/publish-to.js +23 -0
- package/src/nodes/invert-flag/recompute.js +12 -0
- package/src/nodes/invert-flag/reset.js +12 -0
- package/src/nodes/invert-flag/update.js +32 -0
- package/src/nodes/kalman1d/index.js +20 -0
- package/src/nodes/kalman1d/init.js +103 -0
- package/src/nodes/kalman1d/introspect.js +250 -0
- package/src/nodes/kalman1d/publish-to.js +34 -0
- package/src/nodes/kalman1d/recompute.js +23 -0
- package/src/nodes/kalman1d/reset.js +29 -0
- package/src/nodes/kalman1d/update.js +144 -0
- package/src/nodes/kernel/index.js +18 -0
- package/src/nodes/kernel/init.js +66 -0
- package/src/nodes/kernel/introspect.js +163 -0
- package/src/nodes/kernel/presets.js +74 -0
- package/src/nodes/kernel/publish-to.js +25 -0
- package/src/nodes/kernel/recompute.js +14 -0
- package/src/nodes/kernel/reset.js +20 -0
- package/src/nodes/kernel/update.js +69 -0
- package/src/nodes/lag/index.js +21 -0
- package/src/nodes/lag/init.js +87 -0
- package/src/nodes/lag/introspect.js +202 -0
- package/src/nodes/lag/publish-to.js +64 -0
- package/src/nodes/lag/recompute.js +33 -0
- package/src/nodes/lag/reset.js +49 -0
- package/src/nodes/lag/update.js +133 -0
- package/src/nodes/median3/index.js +19 -0
- package/src/nodes/median3/init.js +40 -0
- package/src/nodes/median3/introspect.js +116 -0
- package/src/nodes/median3/publish-to.js +21 -0
- package/src/nodes/median3/recompute.js +12 -0
- package/src/nodes/median3/reset.js +15 -0
- package/src/nodes/median3/update.js +58 -0
- package/src/nodes/moments-digest/copy-from.js +12 -0
- package/src/nodes/moments-digest/flush.js +17 -0
- package/src/nodes/moments-digest/index.js +132 -0
- package/src/nodes/moments-digest/init.js +84 -0
- package/src/nodes/moments-digest/introspect.js +122 -0
- package/src/nodes/moments-digest/prelude.js +28 -0
- package/src/nodes/moments-digest/publish-to.js +31 -0
- package/src/nodes/moments-digest/recompute.js +9 -0
- package/src/nodes/moments-digest/reset.js +20 -0
- package/src/nodes/moments-digest/update.js +182 -0
- package/src/nodes/node-names.js +67 -0
- package/src/nodes/page-hinkley/index.js +20 -0
- package/src/nodes/page-hinkley/init.js +66 -0
- package/src/nodes/page-hinkley/introspect.js +193 -0
- package/src/nodes/page-hinkley/publish-to.js +28 -0
- package/src/nodes/page-hinkley/recompute.js +14 -0
- package/src/nodes/page-hinkley/reset.js +23 -0
- package/src/nodes/page-hinkley/update.js +105 -0
- package/src/nodes/pass-if/index.js +21 -0
- package/src/nodes/pass-if/init.js +37 -0
- package/src/nodes/pass-if/introspect.js +93 -0
- package/src/nodes/pass-if/publish-to.js +15 -0
- package/src/nodes/pass-if/recompute.js +15 -0
- package/src/nodes/pass-if/reset.js +18 -0
- package/src/nodes/pass-if/update.js +37 -0
- package/src/nodes/persist-if/helpers.js +111 -0
- package/src/nodes/persist-if/index.js +9 -0
- package/src/nodes/persist-if/init.js +98 -0
- package/src/nodes/persist-if/introspect.js +106 -0
- package/src/nodes/persist-if/publish-to.js +21 -0
- package/src/nodes/persist-if/recompute.js +20 -0
- package/src/nodes/persist-if/reset.js +35 -0
- package/src/nodes/persist-if/update.js +112 -0
- package/src/nodes/persistence-check/index.js +14 -0
- package/src/nodes/persistence-check/init.js +38 -0
- package/src/nodes/persistence-check/introspect.js +144 -0
- package/src/nodes/persistence-check/publish-to.js +11 -0
- package/src/nodes/persistence-check/recompute.js +7 -0
- package/src/nodes/persistence-check/reset.js +12 -0
- package/src/nodes/persistence-check/update.js +57 -0
- package/src/nodes/process-index/index.js +15 -0
- package/src/nodes/process-index/init.js +48 -0
- package/src/nodes/process-index/introspect.js +182 -0
- package/src/nodes/process-index/publish-to.js +29 -0
- package/src/nodes/process-index/recompute.js +8 -0
- package/src/nodes/process-index/reset.js +8 -0
- package/src/nodes/process-index/update.js +75 -0
- package/src/nodes/ratio/index.js +14 -0
- package/src/nodes/ratio/init.js +37 -0
- package/src/nodes/ratio/introspect.js +155 -0
- package/src/nodes/ratio/publish-to.js +15 -0
- package/src/nodes/ratio/recompute.js +7 -0
- package/src/nodes/ratio/reset.js +7 -0
- package/src/nodes/ratio/update.js +47 -0
- package/src/nodes/sanitize/index.js +82 -0
- package/src/nodes/sanitize/init.js +82 -0
- package/src/nodes/sanitize/introspect.js +194 -0
- package/src/nodes/sanitize/publish-to.js +15 -0
- package/src/nodes/sanitize/recompute.js +8 -0
- package/src/nodes/sanitize/reset.js +14 -0
- package/src/nodes/sanitize/update.js +102 -0
- package/src/nodes/spike-guard/index.js +14 -0
- package/src/nodes/spike-guard/init.js +47 -0
- package/src/nodes/spike-guard/introspect.js +130 -0
- package/src/nodes/spike-guard/publish-to.js +35 -0
- package/src/nodes/spike-guard/recompute.js +14 -0
- package/src/nodes/spike-guard/reset.js +19 -0
- package/src/nodes/spike-guard/update.js +89 -0
- package/src/nodes/sse-emitter/index.js +7 -0
- package/src/nodes/sse-emitter/init.js +105 -0
- package/src/nodes/sse-emitter/introspect.js +134 -0
- package/src/nodes/sse-emitter/publish-to.js +6 -0
- package/src/nodes/sse-emitter/recompute.js +16 -0
- package/src/nodes/sse-emitter/reset.js +19 -0
- package/src/nodes/sse-emitter/update.js +69 -0
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +36 -0
- package/src/nodes/sse-emitter/utils/build-message.js +31 -0
- package/src/nodes/sse-emitter/utils/connection-manager.js +45 -0
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +20 -0
- package/src/nodes/sse-emitter/utils/health-monitor.js +38 -0
- package/src/nodes/sse-emitter/utils/qos-handler.js +41 -0
- package/src/nodes/state-change-detector/helpers.js +35 -0
- package/src/nodes/state-change-detector/index.js +12 -0
- package/src/nodes/state-change-detector/init.js +56 -0
- package/src/nodes/state-change-detector/introspect.js +146 -0
- package/src/nodes/state-change-detector/publish-to.js +20 -0
- package/src/nodes/state-change-detector/recompute.js +6 -0
- package/src/nodes/state-change-detector/reset.js +20 -0
- package/src/nodes/state-change-detector/update.js +97 -0
- package/src/nodes/sw-stats/index.js +14 -0
- package/src/nodes/sw-stats/init.js +51 -0
- package/src/nodes/sw-stats/introspect.js +131 -0
- package/src/nodes/sw-stats/publish-to.js +45 -0
- package/src/nodes/sw-stats/recompute.js +33 -0
- package/src/nodes/sw-stats/reset.js +15 -0
- package/src/nodes/sw-stats/update.js +39 -0
- package/src/nodes/swing-watch/index.js +17 -0
- package/src/nodes/swing-watch/init.js +144 -0
- package/src/nodes/swing-watch/introspect.js +188 -0
- package/src/nodes/swing-watch/publish-to.js +53 -0
- package/src/nodes/swing-watch/recompute.js +22 -0
- package/src/nodes/swing-watch/reset.js +55 -0
- package/src/nodes/swing-watch/update.js +767 -0
- package/src/nodes/tally/index.js +15 -0
- package/src/nodes/tally/init.js +32 -0
- package/src/nodes/tally/introspect.js +116 -0
- package/src/nodes/tally/publish-to.js +28 -0
- package/src/nodes/tally/recompute.js +10 -0
- package/src/nodes/tally/reset.js +10 -0
- package/src/nodes/tally/update.js +44 -0
- package/src/nodes/threshold/index.js +14 -0
- package/src/nodes/threshold/init.js +121 -0
- package/src/nodes/threshold/introspect.js +192 -0
- package/src/nodes/threshold/publish-to.js +11 -0
- package/src/nodes/threshold/recompute.js +8 -0
- package/src/nodes/threshold/reset.js +14 -0
- package/src/nodes/threshold/update.js +54 -0
- package/src/nodes/transform/helpers.js +39 -0
- package/src/nodes/transform/index.js +15 -0
- package/src/nodes/transform/init.js +35 -0
- package/src/nodes/transform/introspect.js +118 -0
- package/src/nodes/transform/publish-to.js +23 -0
- package/src/nodes/transform/recompute.js +12 -0
- package/src/nodes/transform/reset.js +11 -0
- package/src/nodes/transform/update.js +32 -0
- package/src/nodes/trend/compute-confidence.js +112 -0
- package/src/nodes/trend/index.js +14 -0
- package/src/nodes/trend/init.js +81 -0
- package/src/nodes/trend/introspect.js +151 -0
- package/src/nodes/trend/publish-to.js +22 -0
- package/src/nodes/trend/recompute.js +15 -0
- package/src/nodes/trend/reset.js +26 -0
- package/src/nodes/trend/update.js +93 -0
- package/src/nodes/tw-stats/copy-from.js +16 -0
- package/src/nodes/tw-stats/flush.js +12 -0
- package/src/nodes/tw-stats/index.js +41 -0
- package/src/nodes/tw-stats/init.js +92 -0
- package/src/nodes/tw-stats/introspect.js +171 -0
- package/src/nodes/tw-stats/publish-to.js +108 -0
- package/src/nodes/tw-stats/recompute.js +9 -0
- package/src/nodes/tw-stats/reset.js +16 -0
- package/src/nodes/tw-stats/update.js +113 -0
- package/src/nodes/unbalance/index.js +15 -0
- package/src/nodes/unbalance/init.js +59 -0
- package/src/nodes/unbalance/introspect.js +167 -0
- package/src/nodes/unbalance/publish-to.js +52 -0
- package/src/nodes/unbalance/recompute.js +11 -0
- package/src/nodes/unbalance/reset.js +10 -0
- package/src/nodes/unbalance/update.js +83 -0
- package/src/nodes/vector-distance/index.js +14 -0
- package/src/nodes/vector-distance/init.js +54 -0
- package/src/nodes/vector-distance/introspect.js +149 -0
- package/src/nodes/vector-distance/publish-to.js +35 -0
- package/src/nodes/vector-distance/recompute.js +11 -0
- package/src/nodes/vector-distance/reset.js +20 -0
- package/src/nodes/vector-distance/update.js +139 -0
- package/src/nodes/winnow/index.js +12 -0
- package/src/nodes/winnow/init.js +82 -0
- package/src/nodes/winnow/introspect.js +253 -0
- package/src/nodes/winnow/publish-to.js +51 -0
- package/src/nodes/winnow/recompute.js +14 -0
- package/src/nodes/winnow/reset.js +34 -0
- package/src/nodes/winnow/update.js +203 -0
- package/src/tools/training/features.js +227 -0
- package/src/tools/training/index.js +21 -0
- package/src/tools/training/logistic.js +238 -0
- package/src/tools/training/metrics.js +126 -0
- package/src/tools/training/scale.js +239 -0
- package/src/tools/training/validate.js +103 -0
- package/src/windowing/count-sliding/create.js +16 -0
- package/src/windowing/count-sliding/get-reverse-chronological-index.js +19 -0
- package/src/windowing/count-sliding/index.js +6 -0
- package/src/windowing/count-sliding/is-not-full.js +5 -0
- package/src/windowing/count-sliding/push.js +15 -0
- package/src/windowing/count-sliding/reset.js +11 -0
- package/src/windowing/count-sliding/size.js +5 -0
- package/src/windowing/time-sliding/create.js +25 -0
- package/src/windowing/time-sliding/push.js +42 -0
- package/CODE_OF_CONDUCT.md +0 -46
- package/CONTRIBUTING.md +0 -132
- package/docs/.nojekyll +0 -0
- package/docs/CONTRIBUTING.md +0 -134
- package/docs/README.md +0 -132
- package/docs/_coverpage.md +0 -26
- package/docs/_media/bg.png +0 -0
- package/docs/_media/wink-logo.png +0 -0
- package/docs/_navbar.md +0 -14
- package/docs/_sidebar.md +0 -2
- package/docs/index.html +0 -44
- package/eslint.config.js +0 -283
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This function transforms individual nodes into executable closures that automatically
|
|
3
|
+
* handle message processing and flow control.
|
|
4
|
+
*
|
|
5
|
+
* The returned closure executes the node's update/publishTo cycle and automatically
|
|
6
|
+
* routes the processed message to all configured next hops, enabling declarative
|
|
7
|
+
* flow composition from imperative node definitions.
|
|
8
|
+
*
|
|
9
|
+
* Key features:
|
|
10
|
+
* - Enables automatic flow execution without manual sequencing
|
|
11
|
+
* - Provides node-level error isolation to prevent cascade failures
|
|
12
|
+
* - Supports complex routing patterns (fan-out, conditional, parallel)
|
|
13
|
+
* - Built-in error tracking and observability at the node level
|
|
14
|
+
*
|
|
15
|
+
* @param {Object} nodeModule - Node implementation containing init, update, publishTo, etc.
|
|
16
|
+
* @param {number} nodeIndex - Index position in the state store array
|
|
17
|
+
* @param {Array<Function>} nextHops - Array of downstream node closures to invoke
|
|
18
|
+
* @param {Object} [options={}] - Configuration options for error tracking and behavior
|
|
19
|
+
* @param {boolean} [options.trackErrors=true] - Enable error count and details tracking
|
|
20
|
+
* @param {number} [options.maxRecentErrors=5] - Maximum recent errors to retain
|
|
21
|
+
* @returns {Function} Executable closure that processes messages and routes to next hops
|
|
22
|
+
*
|
|
23
|
+
* @example
|
|
24
|
+
* const median3Node = wireNode( median3, [ ewmaFastNode ], 0 );
|
|
25
|
+
* const ewmaNode = wireNode( ewma, [ diffNode ], 1 );
|
|
26
|
+
*
|
|
27
|
+
* // Execute the wired flow
|
|
28
|
+
* median3Node( stateStore, message );
|
|
29
|
+
*
|
|
30
|
+
* // Access error statistics
|
|
31
|
+
* console.log( stateStore[0].errorStats.totalErrors );
|
|
32
|
+
* console.log( stateStore[0].errorStats.recentErrors );
|
|
33
|
+
*/
|
|
34
|
+
const wireNode = function ( nodeModule, nodeIndex, nextHops = [], options = {} ) {
|
|
35
|
+
// Extract options with defaults
|
|
36
|
+
const trackErrors = options.trackErrors !== false; // Default to true
|
|
37
|
+
const maxRecentErrors = options.maxRecentErrors || 5;
|
|
38
|
+
// Validate required node module interface at wire-time
|
|
39
|
+
if ( !nodeModule || typeof nodeModule.update !== 'function' ) {
|
|
40
|
+
throw new TypeError( `composer/wireNode: Node module must implement update function; found ${typeof nodeModule.update}` );
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if ( typeof nodeModule.publishTo !== 'function' ) {
|
|
44
|
+
throw new TypeError( `composer/wireNode: Node module must implement publishTo function; found ${typeof nodeModule.publishTo}` );
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Validate next hops are executable functions
|
|
48
|
+
if ( !Array.isArray( nextHops ) ) {
|
|
49
|
+
throw new TypeError( `composer/wireNode: Next hops must be an array; found ${typeof nextHops}` );
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
for ( let i = 0; i < nextHops.length; i += 1 ) {
|
|
53
|
+
if ( typeof nextHops[ i ] !== 'function' ) {
|
|
54
|
+
throw new TypeError( `composer/wireNode: Next hop at index ${i} must be a function; found ${typeof nextHops[ i ]}` );
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Validate node index for state store access
|
|
59
|
+
if ( typeof nodeIndex !== 'number' || nodeIndex < 0 ) {
|
|
60
|
+
throw new RangeError( `composer/wireNode: Node index must be a non-negative number; found ${nodeIndex}` );
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The wired node closure - executes this node and routes to downstream nodes.
|
|
65
|
+
*
|
|
66
|
+
* This closure maintains the reactive flow execution pattern while providing
|
|
67
|
+
* This closure automatically routes to next hops. It ensures node-level error isolation and
|
|
68
|
+
* maintains referential transparency for functional composition.
|
|
69
|
+
*
|
|
70
|
+
* Includes optional error tracking that maintains statistics directly in the
|
|
71
|
+
* node's state for built-in observability and debugging support.
|
|
72
|
+
*
|
|
73
|
+
* @param {Array} stateStore - Partitioned state array containing node states
|
|
74
|
+
* @param {Object} message - Input message to process through this node
|
|
75
|
+
* @param {Array} [triggers=[]] - Control signal definitions for inter-node high priority communication
|
|
76
|
+
* @returns {void}
|
|
77
|
+
*
|
|
78
|
+
* @throws {Error} Node execution errors are isolated and re-thrown with context
|
|
79
|
+
*/
|
|
80
|
+
const executeWiredNode = function ( stateStore, message ) {
|
|
81
|
+
// Extract this node's state from the partitioned state store
|
|
82
|
+
const nodeState = stateStore[ nodeIndex ];
|
|
83
|
+
|
|
84
|
+
// Initialize error tracking if enabled and not already present
|
|
85
|
+
if ( trackErrors && !nodeState.errorStats ) {
|
|
86
|
+
nodeState.errorStats = {
|
|
87
|
+
totalErrors: 0,
|
|
88
|
+
lastErrorTime: null,
|
|
89
|
+
lastErrorMessage: null,
|
|
90
|
+
recentErrors: [], // Circular buffer of recent error details
|
|
91
|
+
errorsByType: Object.create( null ) // Count by error type
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
// Execute the node's update logic with control signals
|
|
97
|
+
// This modifies the node's internal state based on the incoming message
|
|
98
|
+
const state = nodeModule.update( nodeState, message );
|
|
99
|
+
|
|
100
|
+
// The `update` may return null, typically for filter node.
|
|
101
|
+
if ( state === null ) return;
|
|
102
|
+
|
|
103
|
+
// Publish computed results to the message for downstream consumption
|
|
104
|
+
// This enriches the message with this node's output without side effects
|
|
105
|
+
nodeModule.publishTo( nodeState, message );
|
|
106
|
+
|
|
107
|
+
// Route the enriched message to all configured downstream nodes
|
|
108
|
+
// Each next hop receives the same enriched message and state store
|
|
109
|
+
for ( let i = 0; i < nextHops.length; i += 1 ) {
|
|
110
|
+
nextHops[ i ]( stateStore, message );
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
} catch ( error ) {
|
|
114
|
+
// Track error statistics if enabled
|
|
115
|
+
if ( trackErrors ) {
|
|
116
|
+
const errorStats = nodeState.errorStats;
|
|
117
|
+
const timestamp = Date.now();
|
|
118
|
+
|
|
119
|
+
// Update error counters
|
|
120
|
+
errorStats.totalErrors += 1;
|
|
121
|
+
errorStats.lastErrorTime = timestamp;
|
|
122
|
+
errorStats.lastErrorMessage = error.message;
|
|
123
|
+
|
|
124
|
+
// Track errors by type for pattern analysis
|
|
125
|
+
const errorType = error.constructor.name || 'Error';
|
|
126
|
+
errorStats.errorsByType[ errorType ] = ( errorStats.errorsByType[ errorType ] || 0 ) + 1;
|
|
127
|
+
|
|
128
|
+
// Maintain circular buffer of recent errors
|
|
129
|
+
const errorDetail = {
|
|
130
|
+
timestamp,
|
|
131
|
+
message: error.message,
|
|
132
|
+
type: errorType,
|
|
133
|
+
stack: error.stack ? error.stack.split( '\n' )[ 0 ] : null // First line only
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
// Add to recent errors, maintaining max size
|
|
137
|
+
errorStats.recentErrors.push( errorDetail );
|
|
138
|
+
if ( errorStats.recentErrors.length > maxRecentErrors ) {
|
|
139
|
+
errorStats.recentErrors.shift(); // Remove oldest
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Provide node-level error isolation with contextual information
|
|
144
|
+
// This prevents individual node failures from crashing the entire flow
|
|
145
|
+
const nodeError = new Error( `Node execution failed at index ${nodeIndex}: ${error.message}` );
|
|
146
|
+
nodeError.cause = error;
|
|
147
|
+
nodeError.nodeIndex = nodeIndex;
|
|
148
|
+
nodeError.nodeModule = nodeModule.getNodeType ? nodeModule.getNodeType() : 'unknown';
|
|
149
|
+
|
|
150
|
+
throw nodeError;
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
return executeWiredNode;
|
|
155
|
+
}; // wireNode()
|
|
156
|
+
|
|
157
|
+
export default wireNode;
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
// core/wiring/wire-semantics.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Reads each adapter's `semanticsRequirement` declaration and
|
|
5
|
+
* injects only the slice of `assetClass` that the adapter says it needs.
|
|
6
|
+
*
|
|
7
|
+
* Some adapters need to know about the asset class to do their job — for
|
|
8
|
+
* example QuestDB reads `name`, `columns`, and `insightTypes` to create
|
|
9
|
+
* tables and to know what to write into them. Other adapters (a stdout
|
|
10
|
+
* emitter, an MQTT broker) usually don't care.
|
|
11
|
+
*
|
|
12
|
+
* Each adapter declares what it needs via a `semanticsRequirement` export
|
|
13
|
+
* on its module. The wiring layer reads that declaration and acts on it
|
|
14
|
+
* before calling the adapter's factory:
|
|
15
|
+
*
|
|
16
|
+
* semanticsRequirement: {
|
|
17
|
+
* assetClass: {
|
|
18
|
+
* required: true,
|
|
19
|
+
* fields: [ 'name', 'insightTypes', 'columns' ]
|
|
20
|
+
* }
|
|
21
|
+
* }
|
|
22
|
+
*
|
|
23
|
+
* `required: true` means: if the flow author did not call `.assetClass()`,
|
|
24
|
+
* fail at startup with a clear message — never call the adapter's factory
|
|
25
|
+
* with a missing assetClass.
|
|
26
|
+
*
|
|
27
|
+
* `fields` is an allowlist: the adapter receives a fresh object containing
|
|
28
|
+
* exactly those top-level fields and nothing else. This is the documented
|
|
29
|
+
* top-level dependency surface — anyone reading the adapter's declaration
|
|
30
|
+
* sees exactly which assetClass fields the adapter touches without
|
|
31
|
+
* grep-hunting through its consumption code. ADR-018 calls this
|
|
32
|
+
* "capability-indexed" and explicitly rejects the type-indexed
|
|
33
|
+
* alternative ("storage gets assetClass, others don't") that this
|
|
34
|
+
* mechanism replaces.
|
|
35
|
+
*
|
|
36
|
+
* The slicing is **top-level only**. Adapters that read column-internal
|
|
37
|
+
* facts (QuestDB reads `columns.*.type` and `columns.*.resolution`;
|
|
38
|
+
* terminal reads `columns.*.resolution`) currently express
|
|
39
|
+
* those needs through adapter-side defensive validation in their
|
|
40
|
+
* `createStorage` / `createEmitter` plus adapter-side consumption code.
|
|
41
|
+
* The declarative mechanism for column-internal capability is deferred
|
|
42
|
+
* to a future ADR; it waits for a second adapter with different
|
|
43
|
+
* column-internal needs.
|
|
44
|
+
*
|
|
45
|
+
* Why capability-indexed (declaration on the adapter) instead of
|
|
46
|
+
* type-indexed (a wiring rule like "all storages get assetClass"):
|
|
47
|
+
* because type-indexed rules don't generalise. Today only QuestDB needs
|
|
48
|
+
* assetClass; a future structured emitter that publishes to
|
|
49
|
+
* `${assetClass}/${insightType}` topics would want the same slice. With
|
|
50
|
+
* this declaration mechanism it just opts in — no edit to the wiring
|
|
51
|
+
* layer required.
|
|
52
|
+
*
|
|
53
|
+
* `err.code` (setup-time throws carry a classified code per ADR-018):
|
|
54
|
+
* - `MISSING_ASSET_CLASS` — the adapter declared `assetClass.required: true`
|
|
55
|
+
* but the flow runtime has no assetClass to give. Operator remediation:
|
|
56
|
+
* add `.assetClass(assetClassDef)` to the flow definition. Same `err.code`
|
|
57
|
+
* that QuestDB's createStorage already throws; this layer
|
|
58
|
+
* catches the problem earlier in the lifecycle, before the factory is
|
|
59
|
+
* called.
|
|
60
|
+
*
|
|
61
|
+
* @see ADR-018 (semantic capability declaration)
|
|
62
|
+
* @see ADR-017 (adapters read facts; never make decisions)
|
|
63
|
+
*/
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Build a fresh object containing only the fields the adapter declared it
|
|
67
|
+
* reads.
|
|
68
|
+
*
|
|
69
|
+
* Uses `Object.create( null )` because the keys come from the adapter's
|
|
70
|
+
* declaration array — external to this module — so we treat the result
|
|
71
|
+
* as a runtime-keyed dictionary rather than a fixed-shape struct. This
|
|
72
|
+
* also sidesteps the (small) prototype-pollution surface where a field
|
|
73
|
+
* named `__proto__` would mutate the prototype on a plain `{}` literal.
|
|
74
|
+
* Field names are nominally adapter-controlled, but defensive cost is
|
|
75
|
+
* zero so we take it.
|
|
76
|
+
*
|
|
77
|
+
* Fields that the source assetClass does not have come through as
|
|
78
|
+
* `undefined` (same as if the field were absent on the original object).
|
|
79
|
+
* The adapter sees exactly what it asked for; any gap is its own
|
|
80
|
+
* responsibility to handle.
|
|
81
|
+
*
|
|
82
|
+
* @param {Object} assetClass - the full assetClass object from runtime
|
|
83
|
+
* @param {string[]} fields - the allowlist of top-level field names
|
|
84
|
+
* @returns {Object} a new object containing only the requested fields
|
|
85
|
+
*/
|
|
86
|
+
const sliceAssetClass = function ( assetClass, fields ) {
|
|
87
|
+
const slice = Object.create( null );
|
|
88
|
+
for ( let i = 0; i < fields.length; i += 1 ) {
|
|
89
|
+
const field = fields[ i ];
|
|
90
|
+
slice[ field ] = assetClass[ field ];
|
|
91
|
+
}
|
|
92
|
+
return slice;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Apply an adapter's `semanticsRequirement` to its config object before
|
|
97
|
+
* the factory is called. Mutates `effectiveConfig` in place — the caller
|
|
98
|
+
* passes a freshly-spread copy of user config that we are free to enrich.
|
|
99
|
+
*
|
|
100
|
+
* Behaviour matrix:
|
|
101
|
+
*
|
|
102
|
+
* declaration absent → no-op (returns false)
|
|
103
|
+
* declared, runtime supplied → slice of assetClass injected as
|
|
104
|
+
* effectiveConfig.assetClass; returns true
|
|
105
|
+
* declared+required, missing → throws with err.code MISSING_ASSET_CLASS
|
|
106
|
+
* declared+optional, missing → no injection, no error; returns true
|
|
107
|
+
* (declaration was honored — there was
|
|
108
|
+
* just nothing to inject)
|
|
109
|
+
*
|
|
110
|
+
* Today only the `assetClass` key is recognised within the declaration;
|
|
111
|
+
* the shape is extensible for future capability needs (e.g., a
|
|
112
|
+
* `streamSchema` slice once such a thing exists). When the declaration
|
|
113
|
+
* names a key we don't recognise yet, we ignore it silently — adapters
|
|
114
|
+
* are free to use newer wiring features against older composer
|
|
115
|
+
* installations without breaking.
|
|
116
|
+
*
|
|
117
|
+
* Returns a boolean so callers can tell "did this declaration touch my
|
|
118
|
+
* config?". Used by wire-storages during the transitional window when
|
|
119
|
+
* the legacy hardcoded fallback only fires for adapters that DON'T
|
|
120
|
+
* declare semanticsRequirement.
|
|
121
|
+
*
|
|
122
|
+
* @param {string} adapterId - identifier used in the error message
|
|
123
|
+
* @param {Object} module - the imported adapter module/object
|
|
124
|
+
* @param {Object|null} runtimeAssetClass - the assetClass passed via
|
|
125
|
+
* flow.assetClass(), or null/undefined if the flow author did not call it
|
|
126
|
+
* @param {Object} effectiveConfig - the config object that will be passed
|
|
127
|
+
* to the adapter factory; mutated in place when injection happens
|
|
128
|
+
* @returns {boolean} true if the adapter declared a semanticsRequirement
|
|
129
|
+
* that this helper recognised (whether or not it injected anything);
|
|
130
|
+
* false if no declaration was present
|
|
131
|
+
* @throws {Error} `err.code === 'MISSING_ASSET_CLASS'` when the adapter
|
|
132
|
+
* declared `assetClass.required: true` and runtimeAssetClass is null
|
|
133
|
+
*/
|
|
134
|
+
const applySemanticsRequirement = function ( adapterId, module, runtimeAssetClass, effectiveConfig ) {
|
|
135
|
+
const requirement = module && module.semanticsRequirement;
|
|
136
|
+
|
|
137
|
+
// Adapters that don't read semantics declare nothing (or `{}`). Either
|
|
138
|
+
// way we have nothing to inject and nothing to validate.
|
|
139
|
+
if ( !requirement || !requirement.assetClass ) {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const assetClassReq = requirement.assetClass;
|
|
144
|
+
|
|
145
|
+
if ( !runtimeAssetClass ) {
|
|
146
|
+
if ( assetClassReq.required ) {
|
|
147
|
+
// Per ADR-018, setup-time throws carry a classified err.code.
|
|
148
|
+
// MISSING_ASSET_CLASS (not INVALID_CONFIG) so operator tooling
|
|
149
|
+
// can route on it — the remediation is to add a `.assetClass()`
|
|
150
|
+
// call to the flow, not to fix transport config. Same code as
|
|
151
|
+
// QuestDB's existing createStorage guard; this layer
|
|
152
|
+
// catches it earlier in the lifecycle.
|
|
153
|
+
const err = new Error(
|
|
154
|
+
`WinkComposer/wiring: adapter '${adapterId}' declares semanticsRequirement.assetClass.required ` +
|
|
155
|
+
'but the flow has no .assetClass() — add .assetClass(assetClassDef) to the flow definition'
|
|
156
|
+
);
|
|
157
|
+
err.code = 'MISSING_ASSET_CLASS';
|
|
158
|
+
throw err;
|
|
159
|
+
}
|
|
160
|
+
// Declared but not required, and not supplied — leave the
|
|
161
|
+
// adapter's config alone. The adapter is responsible for handling
|
|
162
|
+
// the no-assetClass case if it opted-in optionally.
|
|
163
|
+
return true;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Slice only the declared fields. Adapter never sees more than it
|
|
167
|
+
// asked for; declaration is the auditable contract.
|
|
168
|
+
const fields = assetClassReq.fields || [];
|
|
169
|
+
effectiveConfig.assetClass = sliceAssetClass( runtimeAssetClass, fields );
|
|
170
|
+
return true;
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
export { applySemanticsRequirement, sliceAssetClass };
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
// core/wiring/wire-storages.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Holds the running storage instances for a flow and creates
|
|
5
|
+
* them on demand from the spec.
|
|
6
|
+
*
|
|
7
|
+
* A "storage" is the thing that persists messages to a time-series database
|
|
8
|
+
* (QuestDB today; potentially others later). One storage per name is enough
|
|
9
|
+
* — many partitions and many persist-if nodes can share it — so we keep
|
|
10
|
+
* them in a small registry keyed by storage name and only build a new one
|
|
11
|
+
* if we haven't seen that name before.
|
|
12
|
+
*
|
|
13
|
+
* This file is responsible for four things:
|
|
14
|
+
* - Building each storage at flow startup, by calling its module's
|
|
15
|
+
* `createStorage(config)` factory and storing the returned handle.
|
|
16
|
+
* The factory may be async (QuestDB needs to negotiate tables via
|
|
17
|
+
* PostgreSQL before the ILP sender is ready); both async and sync
|
|
18
|
+
* factories are supported.
|
|
19
|
+
* - Calling `applySemanticsRequirement` (from `wire-semantics.js`) on each
|
|
20
|
+
* storage so adapters that declare `semanticsRequirement` get the
|
|
21
|
+
* requested slice of `assetClass` injected before their factory is
|
|
22
|
+
* called (capability-indexed injection per ADR-018).
|
|
23
|
+
* The earlier hardcoded "every storage gets the full assetClass
|
|
24
|
+
* plus a tablePrefix default" pattern was removed; adapters that need
|
|
25
|
+
* defaulting (e.g., QuestDB's `tablePrefix ?? assetClass.name`) own
|
|
26
|
+
* that logic themselves.
|
|
27
|
+
* - Checking, the moment we get the handle back, that it has the methods
|
|
28
|
+
* the rest of the framework will call later (fail loudly at startup
|
|
29
|
+
* rather than crash deep in the per-message hot path). See
|
|
30
|
+
* `assert-handle.js` for the check helper and the error format.
|
|
31
|
+
* - Telling the partition manager which storages expose `getPressure()` so
|
|
32
|
+
* the pressure-aware yield decision (ADR-020, Draft) can poll them. The
|
|
33
|
+
* list is exposed via `getBackpressureAwareSinks()` and assembled by
|
|
34
|
+
* `flow/run.js` after the flow's partition state is set up.
|
|
35
|
+
*
|
|
36
|
+
* Restart safety: `shutdown()` clears the registry after stopping every
|
|
37
|
+
* storage, so a follow-up `wire()` call on the same process gets a fresh
|
|
38
|
+
* set of handles. (See the longer comment on shutdown for the production
|
|
39
|
+
* and testing rationale.)
|
|
40
|
+
*
|
|
41
|
+
* The IIFE around the registry hides it from outside callers; only the
|
|
42
|
+
* exported methods can read or change it.
|
|
43
|
+
*
|
|
44
|
+
* @see ADR-018 (sink method surface)
|
|
45
|
+
* @see ADR-020 (backpressure-aware-sinks collection; Draft)
|
|
46
|
+
*/
|
|
47
|
+
|
|
48
|
+
import { assertHandle } from './assert-handle.js';
|
|
49
|
+
import { assertModuleDurability } from './assert-module.js';
|
|
50
|
+
import { applySemanticsRequirement } from './wire-semantics.js';
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Methods the framework calls on a storage handle later, so they have to
|
|
54
|
+
* be present at wire time per ADR-018.
|
|
55
|
+
* `getStats` was previously here and was later removed from the
|
|
56
|
+
* contract's method surface.
|
|
57
|
+
* `getPressure` is optional — its presence decides whether the handle is
|
|
58
|
+
* collected into `backpressureAwareSinks`, not whether the wire succeeds.
|
|
59
|
+
*/
|
|
60
|
+
const REQUIRED_STORAGE_METHODS = [ 'write', 'flush', 'shutdown', 'getHealth' ];
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Serializes a rejection's `dropped` payload for the shutdown log. A
|
|
64
|
+
* third-party adapter can reject with anything — including structures
|
|
65
|
+
* JSON.stringify throws on (circular references). The log line must never
|
|
66
|
+
* throw: a throw here used to skip the registry clear that followed it.
|
|
67
|
+
* Mirrors `wire-emitters.js`.
|
|
68
|
+
*
|
|
69
|
+
* @param {*} dropped - the rejection's dropped payload
|
|
70
|
+
* @returns {string} JSON, or a fallback marker when unserializable
|
|
71
|
+
*/
|
|
72
|
+
const describeDropped = function ( dropped ) {
|
|
73
|
+
try {
|
|
74
|
+
return JSON.stringify( dropped );
|
|
75
|
+
} catch {
|
|
76
|
+
return '[unserializable]';
|
|
77
|
+
}
|
|
78
|
+
}; // describeDropped()
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Stamps the annotate key-sweep support object onto a persistIf spec.
|
|
82
|
+
*
|
|
83
|
+
* The persistIf node warns, once per gate, when a function-form annotate
|
|
84
|
+
* returns a record with invented keys — keys that are neither declared
|
|
85
|
+
* columns nor message fields. Such a key is almost always a typo, and it
|
|
86
|
+
* fails silently: the persist plan writes only declared columns, so the
|
|
87
|
+
* value never reaches storage. The node needs two things for that check,
|
|
88
|
+
* both known only here at wire time: the declared-column set for the
|
|
89
|
+
* spec's insightType, and a once-flag every partition shares. Both ride
|
|
90
|
+
* the spec, exactly like `spec.storage` — `init` copies the reference into
|
|
91
|
+
* each partition's state, so one flag covers all partitions of the gate.
|
|
92
|
+
*
|
|
93
|
+
* No stamp when there is nothing to check: no function-form annotate, no
|
|
94
|
+
* asset class, or an insightType the asset class does not declare (flow
|
|
95
|
+
* validation rejects that last case already; this guard stays defensive).
|
|
96
|
+
*
|
|
97
|
+
* @param {Object} spec - persistIf node spec (mutated: gains annotateSweep)
|
|
98
|
+
* @param {Object|null} assetClass - The flow's asset class, when provided
|
|
99
|
+
*/
|
|
100
|
+
const stampAnnotateSweep = function ( spec, assetClass ) {
|
|
101
|
+
if ( typeof spec.annotate !== 'function' ) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
const insightTypeSpec = assetClass && assetClass.insightTypes && assetClass.insightTypes[ spec.insightType ];
|
|
105
|
+
if ( !insightTypeSpec ) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
spec.annotateSweep = {
|
|
109
|
+
declaredColumns: new Set( insightTypeSpec.columns ),
|
|
110
|
+
checked: false
|
|
111
|
+
};
|
|
112
|
+
}; // stampAnnotateSweep()
|
|
113
|
+
|
|
114
|
+
const storages = ( function () {
|
|
115
|
+
// ========================================================================
|
|
116
|
+
// PRIVATE STATE: Singleton Registry
|
|
117
|
+
// ========================================================================
|
|
118
|
+
// The registry holds all storage singletons keyed by their storage name.
|
|
119
|
+
// This ensures exactly one instance per storage across the entire system,
|
|
120
|
+
// preventing resource waste from duplicate connections/handles.
|
|
121
|
+
const singletons = Object.create( null );
|
|
122
|
+
|
|
123
|
+
// ========================================================================
|
|
124
|
+
// WIRE: Discover and Create Storage Singletons
|
|
125
|
+
// ========================================================================
|
|
126
|
+
// Scans node specifications to find Persist If nodes and creates singleton
|
|
127
|
+
// storages for their targets. This is called during flow initialization
|
|
128
|
+
// for each specialization, but the singleton check ensures each storage
|
|
129
|
+
// is created and initialized exactly once.
|
|
130
|
+
//
|
|
131
|
+
// Storage Lifecycle:
|
|
132
|
+
// 1. createStorage(config) - Creates and initializes storage (may be async)
|
|
133
|
+
// 2. write() - Called during message processing
|
|
134
|
+
// 3. shutdown() - Graceful termination
|
|
135
|
+
//
|
|
136
|
+
// The createStorage() call may be async (e.g., QuestDB needs to create tables
|
|
137
|
+
// via PostgreSQL before the storage is ready). For backwards compatibility,
|
|
138
|
+
// sync createStorage functions are also supported.
|
|
139
|
+
//
|
|
140
|
+
// Legacy init() support: If storage.init() exists, it's called after creation
|
|
141
|
+
// for backwards compatibility with older storage adapters.
|
|
142
|
+
//
|
|
143
|
+
// assetClass: Optional asset class object from flow's .assetClass() method.
|
|
144
|
+
// When provided, it is injected into storage config and tablePrefix defaults
|
|
145
|
+
// to assetClass.name if not explicitly specified.
|
|
146
|
+
const wire = async function ( specs, storageConfigs, storageModules, assetClass = null ) {
|
|
147
|
+
const pendingStorages = [];
|
|
148
|
+
|
|
149
|
+
for ( const spec of specs ) {
|
|
150
|
+
// Skip non-persistIf nodes
|
|
151
|
+
if ( spec.nodeType !== 'Persist If' || !spec.storageName ) {
|
|
152
|
+
continue; // eslint-disable-line no-continue
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const storageName = spec.storageName;
|
|
156
|
+
|
|
157
|
+
// Skip if already in singleton registry
|
|
158
|
+
if ( singletons[ storageName ] ) {
|
|
159
|
+
continue; // eslint-disable-line no-continue
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const module = storageModules[ storageName ];
|
|
163
|
+
if ( !module?.createStorage ) {
|
|
164
|
+
throw new Error( `composer: Invalid storage module: ${storageName}` );
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// ADR-018 module-surface check: the module must say what a
|
|
168
|
+
// crash costs — before the factory runs.
|
|
169
|
+
assertModuleDurability( storageName, module );
|
|
170
|
+
|
|
171
|
+
// Build effective config starting from user config; we layer
|
|
172
|
+
// adapter-driven injection on top.
|
|
173
|
+
const userConfig = storageConfigs[ storageName ] || {};
|
|
174
|
+
const effectiveConfig = { ...userConfig };
|
|
175
|
+
|
|
176
|
+
// Capability-indexed injection per ADR-018. The helper
|
|
177
|
+
// reads `module.semanticsRequirement` and injects the declared
|
|
178
|
+
// slice of assetClass when one is present. Adapters that don't
|
|
179
|
+
// need semantics declare nothing and the helper is a no-op for
|
|
180
|
+
// them. Adapters that do (e.g., QuestDB) declare what they
|
|
181
|
+
// need and only that slice is injected.
|
|
182
|
+
//
|
|
183
|
+
// The earlier hardcoded "every storage gets the full
|
|
184
|
+
// assetClass plus a tablePrefix default" path was removed.
|
|
185
|
+
// Adapters that need a
|
|
186
|
+
// tablePrefix default now own that logic themselves (e.g.,
|
|
187
|
+
// QuestDB's createStorage defaults `tablePrefix` from
|
|
188
|
+
// `assetClass.name` when not supplied).
|
|
189
|
+
applySemanticsRequirement(
|
|
190
|
+
storageName,
|
|
191
|
+
module,
|
|
192
|
+
assetClass,
|
|
193
|
+
effectiveConfig
|
|
194
|
+
);
|
|
195
|
+
|
|
196
|
+
// createStorage may return Promise (async) or storage directly (sync)
|
|
197
|
+
const storageOrPromise = module.createStorage( effectiveConfig );
|
|
198
|
+
pendingStorages.push( { storageName, storageOrPromise } );
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Await all pending storage creations (handles both async and sync)
|
|
202
|
+
// Sequential to ensure proper resource ordering (file locks, ports)
|
|
203
|
+
for ( const { storageName, storageOrPromise } of pendingStorages ) {
|
|
204
|
+
const storage = await storageOrPromise; // eslint-disable-line no-await-in-loop
|
|
205
|
+
|
|
206
|
+
// Wire-time shape check (ADR-018). Fails fast with a
|
|
207
|
+
// descriptive error instead of letting a missing method crash
|
|
208
|
+
// the per-message hot path later.
|
|
209
|
+
assertHandle( storageName, storage, REQUIRED_STORAGE_METHODS );
|
|
210
|
+
|
|
211
|
+
singletons[ storageName ] = storage;
|
|
212
|
+
|
|
213
|
+
// Legacy init() support for backwards compatibility
|
|
214
|
+
if ( storage.init ) {
|
|
215
|
+
await storage.init(); // eslint-disable-line no-await-in-loop
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// Assign storage references to specs, plus the annotate key-sweep
|
|
220
|
+
// support the node reads at init (see stampAnnotateSweep above).
|
|
221
|
+
for ( const spec of specs ) {
|
|
222
|
+
if ( spec.nodeType === 'Persist If' && spec.storageName ) {
|
|
223
|
+
spec.storage = singletons[ spec.storageName ];
|
|
224
|
+
stampAnnotateSweep( spec, assetClass );
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}; // wire()
|
|
228
|
+
|
|
229
|
+
// ========================================================================
|
|
230
|
+
// GET BACKPRESSURE-AWARE SINKS
|
|
231
|
+
// ========================================================================
|
|
232
|
+
/**
|
|
233
|
+
* Returns the subset of wired storages that expose `getPressure()`.
|
|
234
|
+
* Keys are namespaced as `storage:<storageName>` so a debug log at
|
|
235
|
+
* the call site (the pressure-aware yield decision, ADR-020)
|
|
236
|
+
* immediately tells the operator which sink crossed the threshold.
|
|
237
|
+
* See `flow/run.js` for the assembly into
|
|
238
|
+
* `composerState.partitionState.backpressureAwareSinks`.
|
|
239
|
+
*/
|
|
240
|
+
const getBackpressureAwareSinks = function () {
|
|
241
|
+
const map = Object.create( null );
|
|
242
|
+
for ( const [ storageName, handle ] of Object.entries( singletons ) ) {
|
|
243
|
+
if ( typeof handle.getPressure === 'function' ) {
|
|
244
|
+
map[ `storage:${storageName}` ] = handle;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return map;
|
|
248
|
+
}; // getBackpressureAwareSinks()
|
|
249
|
+
|
|
250
|
+
// ========================================================================
|
|
251
|
+
// GET: Retrieve All Singletons (Defensive Copy)
|
|
252
|
+
// ========================================================================
|
|
253
|
+
// Returns a shallow copy of the singleton registry to prevent external
|
|
254
|
+
// mutation. This is useful for health monitoring and debugging.
|
|
255
|
+
const get = function () {
|
|
256
|
+
return { ...singletons };
|
|
257
|
+
}; // get()
|
|
258
|
+
|
|
259
|
+
// ========================================================================
|
|
260
|
+
// SHUTDOWN: Graceful Termination of All Storages
|
|
261
|
+
// ========================================================================
|
|
262
|
+
// Coordinates the shutdown of all storage singletons, ensuring connections
|
|
263
|
+
// are properly closed and buffered data is flushed. Uses Promise.allSettled
|
|
264
|
+
// to ensure all shutdowns are attempted even if some fail, preventing one
|
|
265
|
+
// failed shutdown from blocking others. Returns a Promise that resolves
|
|
266
|
+
// when all shutdown attempts complete (successfully or not).
|
|
267
|
+
//
|
|
268
|
+
// IMPORTANT: Registry Clearing Behavior
|
|
269
|
+
// -------------------------------------
|
|
270
|
+
// After calling shutdown() on all storages, the singleton registry is cleared.
|
|
271
|
+
// This design decision serves two purposes:
|
|
272
|
+
//
|
|
273
|
+
// 1. PRODUCTION: Enables clean restart scenarios. If a pipeline shuts down
|
|
274
|
+
// and needs to restart (e.g., after reconfiguration), fresh storage
|
|
275
|
+
// instances will be created on the next wire() call. Without clearing,
|
|
276
|
+
// stale/closed storage references would persist and cause errors.
|
|
277
|
+
//
|
|
278
|
+
// 2. TESTING: Enables test isolation. Each test can call shutdown() in
|
|
279
|
+
// afterEach() to ensure a clean slate for the next test. Without this,
|
|
280
|
+
// the singleton pattern would cause state leakage between tests.
|
|
281
|
+
//
|
|
282
|
+
// Usage Guidelines:
|
|
283
|
+
// - In production: Call shutdown() only during pipeline termination or
|
|
284
|
+
// when intentionally restarting the pipeline with new configuration.
|
|
285
|
+
// - In tests: Call shutdown() in afterEach() to ensure test isolation.
|
|
286
|
+
// - After shutdown(): Any subsequent wire() calls will create new instances.
|
|
287
|
+
//
|
|
288
|
+
// Per ADR-018, each storage's own shutdown takes `{ timeout }`.
|
|
289
|
+
// We pass it through so the flow's per-stage time budget (set in
|
|
290
|
+
// flow/run.js) reaches every storage the same way.
|
|
291
|
+
const shutdown = async function ( { timeout } = {} ) {
|
|
292
|
+
const opts = ( timeout === undefined ) ? undefined : { timeout };
|
|
293
|
+
const names = Object.keys( singletons );
|
|
294
|
+
const results = await Promise.allSettled(
|
|
295
|
+
names.map( ( name ) => {
|
|
296
|
+
// A non-conforming adapter may THROW from shutdown()
|
|
297
|
+
// instead of rejecting. Convert the throw into this
|
|
298
|
+
// adapter's own rejection slot — escaping here would
|
|
299
|
+
// skip the siblings' drain and the registry clear.
|
|
300
|
+
try {
|
|
301
|
+
return opts ? singletons[ name ].shutdown( opts ) : singletons[ name ].shutdown();
|
|
302
|
+
} catch ( err ) {
|
|
303
|
+
return Promise.reject( err );
|
|
304
|
+
}
|
|
305
|
+
} )
|
|
306
|
+
);
|
|
307
|
+
// Clear the registry FIRST for clean restart capability (see
|
|
308
|
+
// documentation above for the rationale). This must not depend
|
|
309
|
+
// on the logging below — a hostile rejection payload used to
|
|
310
|
+
// throw mid-log and leave the stale handles behind.
|
|
311
|
+
for ( const key of names ) {
|
|
312
|
+
delete singletons[ key ];
|
|
313
|
+
}
|
|
314
|
+
// A rejected drain must reach a human (ADR-018's two-party
|
|
315
|
+
// rule). allSettled keeps one failing storage from blocking its
|
|
316
|
+
// siblings' drain — but the swallowed rejection is a data loss
|
|
317
|
+
// that would otherwise be visible only to callers inspecting the
|
|
318
|
+
// settled results. Log each one, classified, naming the adapter.
|
|
319
|
+
// String( err ) rather than interpolation: a Symbol rejection
|
|
320
|
+
// reason throws inside a template literal.
|
|
321
|
+
results.forEach( ( result, i ) => {
|
|
322
|
+
if ( result.status === 'rejected' ) {
|
|
323
|
+
const err = result.reason || {};
|
|
324
|
+
const dropped = err.dropped ? ` dropped=${describeDropped( err.dropped )}` : '';
|
|
325
|
+
console.error(
|
|
326
|
+
`WinkComposer/wiring: storage '${names[ i ]}' shutdown failed [${err.code || 'UNKNOWN'}]: ${err.message || String( err )}${dropped}`
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
} );
|
|
330
|
+
return results;
|
|
331
|
+
}; // shutdown()
|
|
332
|
+
|
|
333
|
+
// ========================================================================
|
|
334
|
+
// PUBLIC API
|
|
335
|
+
// ========================================================================
|
|
336
|
+
return {
|
|
337
|
+
wire,
|
|
338
|
+
get,
|
|
339
|
+
getBackpressureAwareSinks,
|
|
340
|
+
shutdown
|
|
341
|
+
};
|
|
342
|
+
}() ); // storages()
|
|
343
|
+
|
|
344
|
+
export default storages;
|