@winkjs/composer 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -3
- package/src/composer.js +1 -1
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +108 -633
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +20 -5
- package/src/core/env-vars.js +143 -16
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +276 -83
- package/src/core/source-manager/mqtt/dedup.js +19 -14
- package/src/core/source-manager/mqtt/index.js +33 -6
- package/src/core/source-manager/mqtt/status.js +211 -46
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +592 -827
- package/src/core/storage-manager/questdb/persist-plan.js +147 -162
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +176 -0
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +7 -4
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +14 -5
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +62 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +200 -40
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
package/src/core/env-vars.js
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
import os from 'os';
|
|
6
6
|
|
|
7
|
+
import { classifyAddress, localhostRefusalDetail } from './utils/address/index.js';
|
|
8
|
+
|
|
7
9
|
// ============================================================================
|
|
8
10
|
// ENVIRONMENT VARIABLE DEFINITIONS
|
|
9
11
|
// ============================================================================
|
|
@@ -16,14 +18,36 @@ import os from 'os';
|
|
|
16
18
|
// to nothing intended.
|
|
17
19
|
const rawYieldMs = ( process.env.YIELD_TIME_THRESHOLD_MS ?? '500' ).trim();
|
|
18
20
|
|
|
21
|
+
// The log-level default depends on NODE_ENV, and the ENV_VARS object
|
|
22
|
+
// literal cannot read its own nodeEnv entry while it is being built.
|
|
23
|
+
// So NODE_ENV is resolved once here and used in both places (the same
|
|
24
|
+
// hoisting precedent as rawYieldMs above). Production defaults to
|
|
25
|
+
// 'info' so an edge box does not fill its journal with debug lines;
|
|
26
|
+
// everything else defaults to 'debug' for development visibility.
|
|
27
|
+
// ADR-028 records the logger architecture.
|
|
28
|
+
const rawNodeEnv = ( process.env.NODE_ENV ?? 'development' ).trim();
|
|
29
|
+
|
|
19
30
|
const ENV_VARS = {
|
|
20
31
|
// Core Configuration
|
|
21
32
|
edgeDeviceId: ( process.env.EDGE_DEVICE_ID ?? os.hostname() ).trim(),
|
|
22
|
-
nodeEnv:
|
|
33
|
+
nodeEnv: rawNodeEnv,
|
|
34
|
+
|
|
35
|
+
// Logging (ADR-028). 'logger' picks the transport the facade
|
|
36
|
+
// writes through; 'logLevel' picks the lowest level that prints.
|
|
37
|
+
logger: ( process.env.COMPOSER_LOGGER ?? 'console' ).trim(),
|
|
38
|
+
logLevel: ( process.env.COMPOSER_LOG_LEVEL ?? ( ( rawNodeEnv === 'production' ) ? 'info' : 'debug' ) ).trim(),
|
|
23
39
|
|
|
24
40
|
// Partition Management (see ADR-016)
|
|
25
41
|
maxPartitionsAllowed: parseInt( process.env.COMPOSER_MAX_PARTITIONS_ALLOWED ?? '10000', 10 ),
|
|
26
42
|
|
|
43
|
+
// Fault containment (ADR-018 — the flow runtime owns per-message
|
|
44
|
+
// dispatch failure). This many CONSECUTIVE message failures stop
|
|
45
|
+
// the flow in the terminal 'errored' phase; one success resets
|
|
46
|
+
// the count. The same value caps a partition's consecutive
|
|
47
|
+
// creation failures before it is quarantined. A run of failures
|
|
48
|
+
// means something systemic; one bad message costs only itself.
|
|
49
|
+
messageFailureThreshold: parseInt( process.env.COMPOSER_MESSAGE_FAILURE_THRESHOLD ?? '5', 10 ),
|
|
50
|
+
|
|
27
51
|
// STORAGE_DIR retired 2026-07-09 — the emitter's LevelDB store
|
|
28
52
|
// (removed by ADR-021) was the only thing that ever wrote there.
|
|
29
53
|
// A future WAL adapter defines its own disk-path configuration.
|
|
@@ -41,7 +65,7 @@ const ENV_VARS = {
|
|
|
41
65
|
// `processMessage` (file replays, testHarness, the headless driver):
|
|
42
66
|
// for them the yield tick is the only planned chance for sink flush
|
|
43
67
|
// timers and socket I/O to run. 500 ms is half the tightest background
|
|
44
|
-
// timer it must not starve (
|
|
68
|
+
// timer it must not starve (QUESTDB_FLUSH_INTERVAL_MS, default 1000).
|
|
45
69
|
// Per-flow override: `.yield( { threshold } )`. Infinity = never yield.
|
|
46
70
|
yieldTimeThresholdMs: rawYieldMs === '' ? NaN : Number( rawYieldMs ),
|
|
47
71
|
|
|
@@ -67,12 +91,26 @@ const ENV_VARS = {
|
|
|
67
91
|
mqttSourceDedupWindowMs: parseInt( process.env.MQTT_SOURCE_DEDUP_WINDOW_MS ?? '120000', 10 ),
|
|
68
92
|
mqttSourceDedupMaxEntries: parseInt( process.env.MQTT_SOURCE_DEDUP_MAX_ENTRIES ?? '65536', 10 ),
|
|
69
93
|
|
|
70
|
-
// QuestDB Configuration
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
94
|
+
// QuestDB Configuration. The address defaults are loopback literals,
|
|
95
|
+
// never the name `localhost` (ADR-030). A name can resolve to two
|
|
96
|
+
// addresses, `::1` and `127.0.0.1`, and QuestDB may listen on only
|
|
97
|
+
// one; the run-5 soak lost its write path for four hours that way.
|
|
98
|
+
// The `hostPort` and `mqttUrl` validators below refuse `localhost`
|
|
99
|
+
// outright, so a deployment that sets it stops here, at import.
|
|
100
|
+
questdbIlpUrl: ( process.env.QUESTDB_ILP_URL ?? '127.0.0.1:9000' ).trim(),
|
|
101
|
+
questdbPgUrl: ( process.env.QUESTDB_PG_URL ?? '127.0.0.1:8812' ).trim(),
|
|
102
|
+
// The five legacy flush variables (deprecated, ADR-029; removed in
|
|
103
|
+
// 0.8.0). Each carries a value only when set, so the adapter's
|
|
104
|
+
// DEPRECATED_OPTION line names only what an operator actually set.
|
|
105
|
+
// QUESTDB_IDLE_FLUSH_CHECK_MS maps to QUESTDB_FLUSH_INTERVAL_MS and
|
|
106
|
+
// QUESTDB_AUTO_FLUSH_ROWS to QUESTDB_FLUSH_ROWS; the other three
|
|
107
|
+
// are accepted and ignored.
|
|
108
|
+
questdbFlushMode: process.env.QUESTDB_FLUSH_MODE ?
|
|
109
|
+
process.env.QUESTDB_FLUSH_MODE.trim() : undefined,
|
|
110
|
+
questdbIdleFlushAfterMs: process.env.QUESTDB_IDLE_FLUSH_AFTER_MS ?
|
|
111
|
+
parseInt( process.env.QUESTDB_IDLE_FLUSH_AFTER_MS, 10 ) : undefined,
|
|
112
|
+
questdbIdleFlushCheckMs: process.env.QUESTDB_IDLE_FLUSH_CHECK_MS ?
|
|
113
|
+
parseInt( process.env.QUESTDB_IDLE_FLUSH_CHECK_MS, 10 ) : undefined,
|
|
76
114
|
// Optional QuestDB settings (undefined if not set)
|
|
77
115
|
questdbAutoFlushRows: process.env.QUESTDB_AUTO_FLUSH_ROWS ?
|
|
78
116
|
parseInt( process.env.QUESTDB_AUTO_FLUSH_ROWS, 10 ) : undefined,
|
|
@@ -83,6 +121,31 @@ const ENV_VARS = {
|
|
|
83
121
|
questdbRetryTimeout: process.env.QUESTDB_RETRY_TIMEOUT ?
|
|
84
122
|
parseInt( process.env.QUESTDB_RETRY_TIMEOUT, 10 ) : undefined,
|
|
85
123
|
|
|
124
|
+
// QuestDB flush settings owned by composer (ADR-029). Each carries a
|
|
125
|
+
// value only when the operator set one. The adapter's option
|
|
126
|
+
// resolver supplies the defaults and derives the ceiling and the
|
|
127
|
+
// deadline from the values that won, so a fixed default here would
|
|
128
|
+
// fight a threshold raised in the flow's config.
|
|
129
|
+
questdbFlushRows: process.env.QUESTDB_FLUSH_ROWS ?
|
|
130
|
+
parseInt( process.env.QUESTDB_FLUSH_ROWS, 10 ) : undefined,
|
|
131
|
+
questdbFlushIntervalMs: process.env.QUESTDB_FLUSH_INTERVAL_MS ?
|
|
132
|
+
parseInt( process.env.QUESTDB_FLUSH_INTERVAL_MS, 10 ) : undefined,
|
|
133
|
+
questdbBufferCeilingRows: process.env.QUESTDB_BUFFER_CEILING_ROWS ?
|
|
134
|
+
parseInt( process.env.QUESTDB_BUFFER_CEILING_ROWS, 10 ) : undefined,
|
|
135
|
+
questdbFlushDeadlineMs: process.env.QUESTDB_FLUSH_DEADLINE_MS ?
|
|
136
|
+
parseInt( process.env.QUESTDB_FLUSH_DEADLINE_MS, 10 ) : undefined,
|
|
137
|
+
|
|
138
|
+
// QuestDB transport settings (ADR-029). QUESTDB_STDLIB_HTTP takes
|
|
139
|
+
// the client's own words for `stdlib_http`, `on` or `off`. The
|
|
140
|
+
// adapter's option resolver maps them to a boolean and supplies the
|
|
141
|
+
// default, so each field here carries a value only when set.
|
|
142
|
+
questdbStdlibHttp: process.env.QUESTDB_STDLIB_HTTP ?
|
|
143
|
+
process.env.QUESTDB_STDLIB_HTTP.trim() : undefined,
|
|
144
|
+
questdbRequestTimeout: process.env.QUESTDB_REQUEST_TIMEOUT ?
|
|
145
|
+
parseInt( process.env.QUESTDB_REQUEST_TIMEOUT, 10 ) : undefined,
|
|
146
|
+
questdbInitBufSize: process.env.QUESTDB_INIT_BUF_SIZE ?
|
|
147
|
+
parseInt( process.env.QUESTDB_INIT_BUF_SIZE, 10 ) : undefined,
|
|
148
|
+
|
|
86
149
|
// QuestDB Credentials
|
|
87
150
|
questdbDatabase: ( process.env.QUESTDB_DATABASE ?? 'qdb' ).trim(),
|
|
88
151
|
questdbUser: ( process.env.QUESTDB_USER ?? 'admin' ).trim(),
|
|
@@ -109,6 +172,22 @@ const validators = {
|
|
|
109
172
|
return null;
|
|
110
173
|
},
|
|
111
174
|
|
|
175
|
+
logger: function ( value ) {
|
|
176
|
+
const validTransports = [ 'console', 'json', 'silent' ];
|
|
177
|
+
if ( !validTransports.includes( value ) ) {
|
|
178
|
+
return `Must be one of ${validTransports.join( ', ' )}, got: "${value}"`;
|
|
179
|
+
}
|
|
180
|
+
return null;
|
|
181
|
+
},
|
|
182
|
+
|
|
183
|
+
logLevel: function ( value ) {
|
|
184
|
+
const validLevels = [ 'debug', 'info', 'warn', 'error' ];
|
|
185
|
+
if ( !validLevels.includes( value ) ) {
|
|
186
|
+
return `Must be one of ${validLevels.join( ', ' )}, got: "${value}"`;
|
|
187
|
+
}
|
|
188
|
+
return null;
|
|
189
|
+
},
|
|
190
|
+
|
|
112
191
|
mqttMsgExpiry: function ( value, originalEnv ) {
|
|
113
192
|
if ( isNaN( value ) || value <= 0 ) {
|
|
114
193
|
return `Must be positive integer, got: "${originalEnv}"`;
|
|
@@ -123,17 +202,24 @@ const validators = {
|
|
|
123
202
|
return null;
|
|
124
203
|
},
|
|
125
204
|
|
|
205
|
+
// host:port, with the port required. Accepts a name, an IPv4
|
|
206
|
+
// literal, or a bracketed IPv6 literal (`[::1]:8812`). Refuses
|
|
207
|
+
// `localhost` in every spelling (ADR-030); the shared helper is the
|
|
208
|
+
// one check the adapter schema and factory layers use too.
|
|
126
209
|
hostPort: function ( value ) {
|
|
127
210
|
if ( !value ) return 'Cannot be empty';
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
if ( !hostPortRegex.test( value ) ) {
|
|
211
|
+
const address = classifyAddress( value, 'hostPort' );
|
|
212
|
+
if ( ( address.kind === 'unparsed' ) || ( address.port === undefined ) ) {
|
|
131
213
|
return `Must be host:port format, got: "${value}"`;
|
|
132
214
|
}
|
|
215
|
+
if ( address.kind === 'localhost' ) {
|
|
216
|
+
return localhostRefusalDetail( address );
|
|
217
|
+
}
|
|
133
218
|
return null;
|
|
134
219
|
},
|
|
135
220
|
|
|
136
221
|
questdbFlushMode: function ( value ) {
|
|
222
|
+
if ( value === undefined ) return null;
|
|
137
223
|
const validModes = [ 'auto', 'manual' ];
|
|
138
224
|
if ( !validModes.includes( value ) ) {
|
|
139
225
|
return `Must be one of ${validModes.join( ', ' )}, got: "${value}"`;
|
|
@@ -141,6 +227,15 @@ const validators = {
|
|
|
141
227
|
return null;
|
|
142
228
|
},
|
|
143
229
|
|
|
230
|
+
questdbStdlibHttp: function ( value ) {
|
|
231
|
+
if ( value === undefined ) return null;
|
|
232
|
+
const validWords = [ 'on', 'off' ];
|
|
233
|
+
if ( !validWords.includes( value ) ) {
|
|
234
|
+
return `Must be one of ${validWords.join( ', ' )}, got: "${value}"`;
|
|
235
|
+
}
|
|
236
|
+
return null;
|
|
237
|
+
},
|
|
238
|
+
|
|
144
239
|
positiveIntOrUndefined: function ( value, originalEnv ) {
|
|
145
240
|
if ( value === undefined ) return null;
|
|
146
241
|
if ( isNaN( value ) || value <= 0 ) {
|
|
@@ -156,6 +251,14 @@ const validators = {
|
|
|
156
251
|
return null;
|
|
157
252
|
},
|
|
158
253
|
|
|
254
|
+
nonNegativeIntOrUndefined: function ( value, originalEnv ) {
|
|
255
|
+
if ( value === undefined ) return null;
|
|
256
|
+
if ( isNaN( value ) || value < 0 ) {
|
|
257
|
+
return `Must be non-negative integer, got: "${originalEnv}"`;
|
|
258
|
+
}
|
|
259
|
+
return null;
|
|
260
|
+
},
|
|
261
|
+
|
|
159
262
|
nonNegativeNumberOrInfinity: function ( value, originalEnv ) {
|
|
160
263
|
if ( isNaN( value ) || value < 0 ) {
|
|
161
264
|
return `Must be a non-negative number or Infinity, got: "${originalEnv}"`;
|
|
@@ -170,11 +273,18 @@ const validators = {
|
|
|
170
273
|
return null;
|
|
171
274
|
},
|
|
172
275
|
|
|
276
|
+
// A broker URL. The scheme check comes first; then `localhost` is
|
|
277
|
+
// refused (ADR-030). A URL the parser cannot read is left to the
|
|
278
|
+
// MQTT library, which reports its own error.
|
|
173
279
|
mqttUrl: function ( value ) {
|
|
174
280
|
if ( !value ) return 'Cannot be empty';
|
|
175
281
|
if ( !value.startsWith( 'mqtt://' ) && !value.startsWith( 'mqtts://' ) ) {
|
|
176
282
|
return `Must start with 'mqtt://' or 'mqtts://', got: "${value}"`;
|
|
177
283
|
}
|
|
284
|
+
const address = classifyAddress( value, 'url' );
|
|
285
|
+
if ( address.kind === 'localhost' ) {
|
|
286
|
+
return localhostRefusalDetail( address );
|
|
287
|
+
}
|
|
178
288
|
return null;
|
|
179
289
|
}
|
|
180
290
|
};
|
|
@@ -185,7 +295,10 @@ const validators = {
|
|
|
185
295
|
const validationConfig = [
|
|
186
296
|
{ field: 'edgeDeviceId', validator: validators.edgeDeviceId },
|
|
187
297
|
{ field: 'nodeEnv', validator: validators.nodeEnv },
|
|
298
|
+
{ field: 'logger', validator: validators.logger, label: 'COMPOSER_LOGGER' },
|
|
299
|
+
{ field: 'logLevel', validator: validators.logLevel, label: 'COMPOSER_LOG_LEVEL' },
|
|
188
300
|
{ field: 'maxPartitionsAllowed', validator: validators.positiveInt, originalEnv: 'COMPOSER_MAX_PARTITIONS_ALLOWED' },
|
|
301
|
+
{ field: 'messageFailureThreshold', validator: validators.positiveInt, originalEnv: 'COMPOSER_MESSAGE_FAILURE_THRESHOLD' },
|
|
189
302
|
{ field: 'yieldTimeThresholdMs', validator: validators.nonNegativeNumberOrInfinity, originalEnv: 'YIELD_TIME_THRESHOLD_MS' },
|
|
190
303
|
{ field: 'mqttBrokerUrl', validator: validators.mqttUrl, label: 'MQTT_BROKER_URL' },
|
|
191
304
|
{ field: 'mqttMsgExpiry', validator: validators.mqttMsgExpiry, originalEnv: 'MQTT_MSG_EXPIRY' },
|
|
@@ -201,12 +314,19 @@ const validationConfig = [
|
|
|
201
314
|
{ field: 'questdbIlpUrl', validator: validators.hostPort, label: 'QUESTDB_ILP_URL' },
|
|
202
315
|
{ field: 'questdbPgUrl', validator: validators.hostPort, label: 'QUESTDB_PG_URL' },
|
|
203
316
|
{ field: 'questdbFlushMode', validator: validators.questdbFlushMode, label: 'QUESTDB_FLUSH_MODE' },
|
|
204
|
-
{ field: 'questdbIdleFlushAfterMs', validator: validators.
|
|
205
|
-
{ field: 'questdbIdleFlushCheckMs', validator: validators.
|
|
317
|
+
{ field: 'questdbIdleFlushAfterMs', validator: validators.nonNegativeIntOrUndefined, originalEnv: 'QUESTDB_IDLE_FLUSH_AFTER_MS' },
|
|
318
|
+
{ field: 'questdbIdleFlushCheckMs', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_IDLE_FLUSH_CHECK_MS' },
|
|
206
319
|
{ field: 'questdbAutoFlushRows', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_AUTO_FLUSH_ROWS' },
|
|
207
320
|
{ field: 'questdbAutoFlushIntervalMs', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_AUTO_FLUSH_INTERVAL_MS' },
|
|
208
321
|
{ field: 'questdbMaxBufSize', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_MAX_BUF_SIZE' },
|
|
209
322
|
{ field: 'questdbRetryTimeout', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_RETRY_TIMEOUT' },
|
|
323
|
+
{ field: 'questdbFlushRows', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_FLUSH_ROWS' },
|
|
324
|
+
{ field: 'questdbFlushIntervalMs', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_FLUSH_INTERVAL_MS' },
|
|
325
|
+
{ field: 'questdbBufferCeilingRows', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_BUFFER_CEILING_ROWS' },
|
|
326
|
+
{ field: 'questdbFlushDeadlineMs', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_FLUSH_DEADLINE_MS' },
|
|
327
|
+
{ field: 'questdbStdlibHttp', validator: validators.questdbStdlibHttp, label: 'QUESTDB_STDLIB_HTTP' },
|
|
328
|
+
{ field: 'questdbRequestTimeout', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_REQUEST_TIMEOUT' },
|
|
329
|
+
{ field: 'questdbInitBufSize', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_INIT_BUF_SIZE' },
|
|
210
330
|
// QuestDB Credentials
|
|
211
331
|
{ field: 'questdbDatabase', validator: validators.nonEmptyString, label: 'QUESTDB_DATABASE' },
|
|
212
332
|
{ field: 'questdbUser', validator: validators.nonEmptyString, label: 'QUESTDB_USER' }
|
|
@@ -223,7 +343,14 @@ const validate = function () {
|
|
|
223
343
|
for ( const config of validationConfig ) {
|
|
224
344
|
const value = ENV_VARS[ config.field ];
|
|
225
345
|
const originalEnv = config.originalEnv ? process.env[ config.originalEnv ] : value;
|
|
226
|
-
|
|
346
|
+
// The label is the variable name as the operator typed it. A row
|
|
347
|
+
// that records the variable uses it; otherwise the name is
|
|
348
|
+
// derived from the field, inserting the underscores BEFORE
|
|
349
|
+
// uppercasing (the other order finds no boundaries and printed
|
|
350
|
+
// QUESTDBRETRYTIMEOUT until 2026-09-05).
|
|
351
|
+
const label = config.label ||
|
|
352
|
+
config.originalEnv ||
|
|
353
|
+
config.field.replace( /([a-z])([A-Z])/g, '$1_$2' ).toUpperCase();
|
|
227
354
|
|
|
228
355
|
const error = config.validator( value, originalEnv || value, label );
|
|
229
356
|
if ( error ) {
|
|
@@ -233,8 +360,8 @@ const validate = function () {
|
|
|
233
360
|
|
|
234
361
|
// Fail fast with clear error message
|
|
235
362
|
if ( errors.length > 0 ) {
|
|
236
|
-
console.error( '
|
|
237
|
-
errors.forEach( ( err ) => console.error( ` - ${err}` ) );
|
|
363
|
+
console.error( 'winkComposer/envVars: Environment variable validation failed:' ); // eslint-disable-line no-console -- bootstrap failure runs before any logger can exist
|
|
364
|
+
errors.forEach( ( err ) => console.error( ` - ${err}` ) ); // eslint-disable-line no-console -- bootstrap failure runs before any logger can exist
|
|
238
365
|
process.exit( 1 ); // eslint-disable-line no-process-exit
|
|
239
366
|
}
|
|
240
367
|
}; // validate()
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// core/logger/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The framework's logging facade. Every production
|
|
5
|
+
* log line goes through one small object with four methods: debug,
|
|
6
|
+
* info, warn, and error. Each method takes a message string and an
|
|
7
|
+
* optional flat fields object. The transport behind the facade —
|
|
8
|
+
* where a record actually goes — is picked once at startup from
|
|
9
|
+
* COMPOSER_LOGGER, and the lowest level that prints is picked from
|
|
10
|
+
* COMPOSER_LOG_LEVEL. ADR-028 records the design.
|
|
11
|
+
*
|
|
12
|
+
* Three guarantees shape the code:
|
|
13
|
+
*
|
|
14
|
+
* - **Zero cost when suppressed.** Levels below the threshold are
|
|
15
|
+
* resolved at build time: every suppressed method IS one shared
|
|
16
|
+
* empty function. No comparison, no allocation, no argument
|
|
17
|
+
* inspection happens on a suppressed call. JavaScript still
|
|
18
|
+
* evaluates the arguments themselves, so a hot-path call site
|
|
19
|
+
* guards a fields literal with the precomputed `debugOn` or
|
|
20
|
+
* `infoOn` boolean instead of relying on the no-op.
|
|
21
|
+
* - **A log call never throws.** Logging exists to report faults,
|
|
22
|
+
* so it must not create them. A broken transport, a bad level
|
|
23
|
+
* name, or unserializable fields each degrade to a working
|
|
24
|
+
* console line. The last-resort line interpolates nothing.
|
|
25
|
+
* - **Stream identity.** The console transport mirrors each level
|
|
26
|
+
* to the same console method a raw call would have used, and the
|
|
27
|
+
* json transport keeps the same stderr/stdout split. A line
|
|
28
|
+
* never moves between streams because the facade arrived.
|
|
29
|
+
*
|
|
30
|
+
* There is deliberately no file transport: journald, kubelet, and
|
|
31
|
+
* logrotate own files in every deployment shape (ADR-028 records
|
|
32
|
+
* the rejection and the escape hatches).
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { ENV_VARS } from '../env-vars.js';
|
|
36
|
+
|
|
37
|
+
// Level order: a method emits when its rank is at or above the
|
|
38
|
+
// configured threshold. Internal constant keys — a plain literal.
|
|
39
|
+
const LEVELS = { debug: 10, info: 20, warn: 30, error: 40 };
|
|
40
|
+
|
|
41
|
+
// Which console method carries each level. error and warn go to
|
|
42
|
+
// stderr, info and debug to stdout — the same split raw calls had.
|
|
43
|
+
const CONSOLE_METHODS = { debug: 'log', info: 'log', warn: 'warn', error: 'error' };
|
|
44
|
+
|
|
45
|
+
// The one shared no-op every suppressed method resolves to.
|
|
46
|
+
const NO_OP = function () { /* the shared suppressed-level sink */ };
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Stringify a fields object without ever throwing. Circular or
|
|
50
|
+
* otherwise unserializable fields become a quoted placeholder, so
|
|
51
|
+
* the surrounding line stays valid.
|
|
52
|
+
*
|
|
53
|
+
* @param {*} value - The fields value to render.
|
|
54
|
+
* @returns {string} JSON text, or the placeholder literal.
|
|
55
|
+
*/
|
|
56
|
+
const safeStringify = function ( value ) {
|
|
57
|
+
try {
|
|
58
|
+
return JSON.stringify( value );
|
|
59
|
+
} catch {
|
|
60
|
+
return '"[unserializable fields]"';
|
|
61
|
+
}
|
|
62
|
+
}; // safeStringify()
|
|
63
|
+
|
|
64
|
+
const consoleTransport = {
|
|
65
|
+
emit: function ( level, msg, fields ) {
|
|
66
|
+
const line = ( fields === undefined ) ? msg : ( msg + ' ' + safeStringify( fields ) );
|
|
67
|
+
console[ CONSOLE_METHODS[ level ] ]( line ); // eslint-disable-line no-console -- the console transport IS the console boundary
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const jsonTransport = {
|
|
72
|
+
emit: function ( level, msg, fields ) {
|
|
73
|
+
let line;
|
|
74
|
+
try {
|
|
75
|
+
// JSON.stringify drops a fields key holding undefined,
|
|
76
|
+
// so the no-fields record needs no separate branch.
|
|
77
|
+
line = JSON.stringify( { level, msg, fields } );
|
|
78
|
+
} catch {
|
|
79
|
+
line = JSON.stringify( { level, msg, fields: '[unserializable fields]' } );
|
|
80
|
+
}
|
|
81
|
+
console[ CONSOLE_METHODS[ level ] ]( line ); // eslint-disable-line no-console -- the json transport IS the console boundary
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const silentTransport = {
|
|
86
|
+
emit: NO_OP
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// The env-selectable transports. The mqtt transport joins this map
|
|
90
|
+
// when ADR-031 lands.
|
|
91
|
+
const transports = {
|
|
92
|
+
console: consoleTransport,
|
|
93
|
+
json: jsonTransport,
|
|
94
|
+
silent: silentTransport
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Create an in-memory transport for specs. Each instance records
|
|
99
|
+
* `{ level, msg, fields }` per emission and can be reset. Never
|
|
100
|
+
* selectable through COMPOSER_LOGGER.
|
|
101
|
+
*
|
|
102
|
+
* @returns {object} `{ emit, records, reset }`.
|
|
103
|
+
*/
|
|
104
|
+
const createMemoryTransport = function () {
|
|
105
|
+
const records = [];
|
|
106
|
+
return {
|
|
107
|
+
records,
|
|
108
|
+
reset: function () {
|
|
109
|
+
records.length = 0;
|
|
110
|
+
},
|
|
111
|
+
emit: function ( level, msg, fields ) {
|
|
112
|
+
records.push( { level, msg, fields } );
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
}; // createMemoryTransport()
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Build a logger over a transport at a level. Methods are
|
|
119
|
+
* precompiled here: suppressed levels become the shared no-op, and
|
|
120
|
+
* active levels close over the transport. Invalid inputs fall back
|
|
121
|
+
* to a working console logger with one warning line — building a
|
|
122
|
+
* logger never throws, because logging must not crash its host.
|
|
123
|
+
*
|
|
124
|
+
* @param {object} transport - An object with `emit( level, msg, fields )`.
|
|
125
|
+
* @param {string} levelName - One of debug, info, warn, error.
|
|
126
|
+
* @returns {object} `{ debug, info, warn, error, debugOn, infoOn, level }`.
|
|
127
|
+
*/
|
|
128
|
+
const buildLogger = function ( transport, levelName ) {
|
|
129
|
+
let safeTransport = transport;
|
|
130
|
+
if ( !transport || typeof transport.emit !== 'function' ) {
|
|
131
|
+
console.warn( 'winkComposer/logger: transport has no emit function — falling back to console' ); // eslint-disable-line no-console -- init fallback runs before a working transport exists
|
|
132
|
+
safeTransport = consoleTransport;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
let safeLevel = levelName;
|
|
136
|
+
if ( LEVELS[ safeLevel ] === undefined ) {
|
|
137
|
+
console.warn( 'winkComposer/logger: unknown log level — falling back to info' ); // eslint-disable-line no-console -- init fallback runs before a working transport exists
|
|
138
|
+
safeLevel = 'info';
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const threshold = LEVELS[ safeLevel ];
|
|
142
|
+
|
|
143
|
+
const makeMethod = function ( name ) {
|
|
144
|
+
if ( LEVELS[ name ] < threshold ) {
|
|
145
|
+
return NO_OP;
|
|
146
|
+
}
|
|
147
|
+
return function ( msg, fields ) {
|
|
148
|
+
try {
|
|
149
|
+
safeTransport.emit( name, msg, fields );
|
|
150
|
+
} catch {
|
|
151
|
+
// Last resort, interpolation-free: a broken
|
|
152
|
+
// transport must not crash the host process.
|
|
153
|
+
console.error( 'winkComposer/logger: transport emit failed — record dropped' ); // eslint-disable-line no-console -- last resort when the transport itself threw
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
debug: makeMethod( 'debug' ),
|
|
160
|
+
info: makeMethod( 'info' ),
|
|
161
|
+
warn: makeMethod( 'warn' ),
|
|
162
|
+
error: makeMethod( 'error' ),
|
|
163
|
+
debugOn: ( threshold <= LEVELS.debug ),
|
|
164
|
+
infoOn: ( threshold <= LEVELS.info ),
|
|
165
|
+
level: safeLevel
|
|
166
|
+
};
|
|
167
|
+
}; // buildLogger()
|
|
168
|
+
|
|
169
|
+
// The default instance every production module imports. ENV_VARS
|
|
170
|
+
// validated both values at startup, so the fallbacks above are a
|
|
171
|
+
// second line of defense, not the expected path.
|
|
172
|
+
const logger = buildLogger( transports[ ENV_VARS.logger ], ENV_VARS.logLevel );
|
|
173
|
+
|
|
174
|
+
export { logger, buildLogger, createMemoryTransport, transports };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// core/mqtt-protocol.js
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @fileoverview Shared MQTT protocol invariants for
|
|
4
|
+
* @fileoverview Shared MQTT protocol invariants for winkComposer.
|
|
5
5
|
*
|
|
6
6
|
* These are design decisions, NOT deployment-configurable values.
|
|
7
7
|
* Both source and emitter depend on identical values for interoperability:
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
// ============================================================================
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
* MQTT v5 user property keys for
|
|
21
|
+
* MQTT v5 user property keys for winkComposer metadata.
|
|
22
22
|
* Publisher (emitter) writes these; subscriber (source) reads them.
|
|
23
23
|
*
|
|
24
24
|
* @type {Object}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// core/partition-manager/get-stats.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Read-only statistics snapshot for the partition
|
|
5
|
+
* manager. Lets an operator count dropped messages and partitions
|
|
6
|
+
* instead of scraping log lines (logger-facade epic, Story 6). The
|
|
7
|
+
* counter itself is initialized in init.js and maintained at the
|
|
8
|
+
* drop site in update.js; this file only reads.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Build a snapshot of the partition manager's counters. Cold path:
|
|
13
|
+
* allocates one small object per call and is never called per
|
|
14
|
+
* message.
|
|
15
|
+
*
|
|
16
|
+
* @param {Object} composerState - State object created by init().
|
|
17
|
+
* @returns {{droppedUnknownSpecialization: number,
|
|
18
|
+
* totalPartitionsCreated: number, activePartitions: number}}
|
|
19
|
+
* Messages dropped for an unknown specialization; partition
|
|
20
|
+
* creations attempted, accepted or cap-rejected (ADR-016); and
|
|
21
|
+
* partitions currently held.
|
|
22
|
+
*/
|
|
23
|
+
const getStats = function ( composerState ) {
|
|
24
|
+
return {
|
|
25
|
+
droppedUnknownSpecialization: composerState.droppedUnknownSpecialization,
|
|
26
|
+
totalPartitionsCreated: composerState.totalPartitionsCreated,
|
|
27
|
+
activePartitions: composerState.partitionSpecializations.size
|
|
28
|
+
};
|
|
29
|
+
}; // getStats()
|
|
30
|
+
|
|
31
|
+
export default getStats;
|
|
@@ -3,5 +3,6 @@ export { default as update } from './update.js';
|
|
|
3
3
|
export { default as reset } from './reset.js';
|
|
4
4
|
export { default as recompute } from './recompute.js';
|
|
5
5
|
export { default as publishTo } from './publish-to.js';
|
|
6
|
+
export { default as getStats } from './get-stats.js';
|
|
6
7
|
// Re-export everything from introspection.js
|
|
7
8
|
export * from './introspect.js';
|
|
@@ -18,6 +18,21 @@ const init = function ( flow ) {
|
|
|
18
18
|
// rejection count. See ADR-016.
|
|
19
19
|
composerState.totalPartitionsCreated = 0;
|
|
20
20
|
|
|
21
|
+
// Messages dropped because their specialization field named no
|
|
22
|
+
// known specialization (the update.js drop site). A lifetime
|
|
23
|
+
// counter, read through getStats() — operators count drops
|
|
24
|
+
// instead of scraping log lines.
|
|
25
|
+
composerState.droppedUnknownSpecialization = 0;
|
|
26
|
+
|
|
27
|
+
// Per-partition creation-failure ledgers (ADR-018 fault
|
|
28
|
+
// containment): keyed by each partition's specializedGraphs
|
|
29
|
+
// object, holding { failures, quarantined }. A WeakMap on object
|
|
30
|
+
// identity — specialization names are user-controlled strings, so
|
|
31
|
+
// no in-object key is collision-safe, and the ledger dies with its
|
|
32
|
+
// partition entry automatically. Entries allocate only when a
|
|
33
|
+
// creation fails; the healthy hot path never touches this.
|
|
34
|
+
composerState.creationFailures = new WeakMap();
|
|
35
|
+
|
|
21
36
|
// Yield state (ADR-024): update() sets `yieldPending` when the time
|
|
22
37
|
// threshold is crossed; the flow runtime clears it and takes the
|
|
23
38
|
// event-loop breath after running the message's pipeline.
|