@winkjs/composer 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -3
- package/src/composer.js +1 -1
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +108 -633
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +20 -5
- package/src/core/env-vars.js +143 -16
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +276 -83
- package/src/core/source-manager/mqtt/dedup.js +19 -14
- package/src/core/source-manager/mqtt/index.js +33 -6
- package/src/core/source-manager/mqtt/status.js +211 -46
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +592 -827
- package/src/core/storage-manager/questdb/persist-plan.js +147 -162
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +176 -0
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +7 -4
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +14 -5
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +62 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +200 -40
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
// core/emitter-manager/mqtt/resolve-config.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Resolves and validates the MQTT emitter's configuration
|
|
5
|
+
* before the factory opens anything (ADR-018, fail-fast setup).
|
|
6
|
+
*
|
|
7
|
+
* The factory in `emitter.js` calls `resolveConfig` once. Every
|
|
8
|
+
* refusal here is a classified `INVALID_CONFIG` TypeError. The checks
|
|
9
|
+
* run in one fixed order: the broker URL, the address policy, the
|
|
10
|
+
* codec, the three callbacks, the grace, the queue size, and the will.
|
|
11
|
+
* The factory is async, so a caller sees each refusal as a rejection.
|
|
12
|
+
*
|
|
13
|
+
* The module also arms the three notification callbacks through the
|
|
14
|
+
* shared callback guard. A user throw inside the publish ack chain
|
|
15
|
+
* then becomes one `CALLBACK_FAILED` line instead of a fault inside
|
|
16
|
+
* mqtt.js (ADR-018, ADR-027).
|
|
17
|
+
*
|
|
18
|
+
* Nothing here runs on the message path. Every value is resolved once
|
|
19
|
+
* and captured by the closures the other modules build.
|
|
20
|
+
*
|
|
21
|
+
* @see ADR-018
|
|
22
|
+
* @see ADR-021
|
|
23
|
+
* @see ADR-030
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import mqtt from 'mqtt';
|
|
27
|
+
import { ENV_VARS } from '../../env-vars.js';
|
|
28
|
+
import {
|
|
29
|
+
QOS,
|
|
30
|
+
MQTT_CONFIG,
|
|
31
|
+
DEFAULT_MAX_QUEUE_SIZE,
|
|
32
|
+
MQTT_INFLIGHT_ID_LIMIT
|
|
33
|
+
} from './constants.js';
|
|
34
|
+
import { wrapCallback } from '../../utils/callback-guard/index.js';
|
|
35
|
+
import { jitteredPeriod } from '../../utils/jitter/index.js';
|
|
36
|
+
import { logger } from '../../logger/index.js';
|
|
37
|
+
import { classifyAddress, localhostRefusalMessage } from '../../utils/address/index.js';
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Build a config-error TypeError tagged for ADR-018 fail-fast setup routing.
|
|
41
|
+
* @param {string} message
|
|
42
|
+
* @returns {TypeError}
|
|
43
|
+
*/
|
|
44
|
+
export const invalidConfig = function ( message ) {
|
|
45
|
+
const err = new TypeError( 'winkComposer/mqttEmitter: ' + message );
|
|
46
|
+
err.code = 'INVALID_CONFIG';
|
|
47
|
+
return err;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Classifies the broker address and refuses `localhost` (ADR-030).
|
|
52
|
+
* The schema already refused it at flow definition; this call covers
|
|
53
|
+
* the `MQTT_BROKER_URL` fallback and direct callers, and carries the
|
|
54
|
+
* classified code.
|
|
55
|
+
*
|
|
56
|
+
* @param {string} brokerUrl - The broker URL, config or env fallback
|
|
57
|
+
* @returns {Object} The classified address
|
|
58
|
+
* @throws {TypeError} INVALID_CONFIG when the host is `localhost`
|
|
59
|
+
*/
|
|
60
|
+
const assertBrokerNotLocalhost = function ( brokerUrl ) {
|
|
61
|
+
const address = classifyAddress( brokerUrl, 'url' );
|
|
62
|
+
if ( address.kind === 'localhost' ) {
|
|
63
|
+
throw invalidConfig( localhostRefusalMessage( { field: 'brokerUrl', address, envVar: 'MQTT_BROKER_URL' } ) );
|
|
64
|
+
}
|
|
65
|
+
return address;
|
|
66
|
+
}; // assertBrokerNotLocalhost()
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Throw classified when an optional callback option is set to a non-function.
|
|
70
|
+
* @param {*} value - the configured option value
|
|
71
|
+
* @param {string} name - the option name for the error message
|
|
72
|
+
*/
|
|
73
|
+
const assertOptionalCallback = function ( value, name ) {
|
|
74
|
+
if ( ( value !== undefined ) && ( typeof value !== 'function' ) ) {
|
|
75
|
+
throw invalidConfig( `${name} must be a function` );
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Validate and build the mqtt.js will (last-will testament) options.
|
|
81
|
+
*
|
|
82
|
+
* Direct createEmitter callers bypass the DSL schema, so the factory
|
|
83
|
+
* checks the will shape itself, and the payload encoding runs behind
|
|
84
|
+
* the same classified-throw rule as every other setup failure
|
|
85
|
+
* (ADR-018, fail-fast setup) — a raw codec TypeError would reach the
|
|
86
|
+
* caller unclassified.
|
|
87
|
+
*
|
|
88
|
+
* @param {Object} will - config.will as supplied
|
|
89
|
+
* @param {Object} codec - config.codec (pack + contentType)
|
|
90
|
+
* @returns {Object} the mqtt.js connect-options `will` object
|
|
91
|
+
*/
|
|
92
|
+
const buildWillOptions = function ( will, codec ) {
|
|
93
|
+
if ( ( typeof will.topic !== 'string' ) || ( will.topic.length === 0 ) ) {
|
|
94
|
+
throw invalidConfig( 'will.topic is required — a non-empty string' );
|
|
95
|
+
}
|
|
96
|
+
if ( ( will.message === undefined ) || ( will.message === null ) ) {
|
|
97
|
+
throw invalidConfig( 'will.message is required' );
|
|
98
|
+
}
|
|
99
|
+
let willPayload;
|
|
100
|
+
try {
|
|
101
|
+
willPayload = Buffer.from( codec.pack( will.message ) );
|
|
102
|
+
} catch ( packErr ) {
|
|
103
|
+
throw invalidConfig( `will.message could not be encoded by the codec: ${packErr.message}` );
|
|
104
|
+
}
|
|
105
|
+
const willOptions = {
|
|
106
|
+
topic: will.topic,
|
|
107
|
+
payload: willPayload,
|
|
108
|
+
// `??`, not `||`: 0 is a valid QoS.
|
|
109
|
+
qos: will.qos ?? QOS,
|
|
110
|
+
retain: will.retain !== false, // Default true
|
|
111
|
+
properties: {
|
|
112
|
+
contentType: codec.contentType
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
if ( codec.payloadFormatIndicator === 1 ) {
|
|
116
|
+
willOptions.properties.payloadFormatIndicator = true;
|
|
117
|
+
}
|
|
118
|
+
return willOptions;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Resolves the broker URL from config or the env fallback, refuses an
|
|
123
|
+
* empty one, and applies the address policy.
|
|
124
|
+
*
|
|
125
|
+
* `??` so explicit '' is the user's choice (not "fall back to env");
|
|
126
|
+
* `.trim()` rejects whitespace-only. Symmetric with QuestDB, which
|
|
127
|
+
* throws on `ilpUrl: ''`. Broker URLs may carry credentials
|
|
128
|
+
* (mqtt://user:pass@host), so the redacted form is built here for
|
|
129
|
+
* every later log line.
|
|
130
|
+
*
|
|
131
|
+
* @param {Object} config - emitter config
|
|
132
|
+
* @returns {{brokerUrl: string, brokerAddress: Object, redactedBrokerUrl: string}}
|
|
133
|
+
*/
|
|
134
|
+
const resolveBrokerUrl = function ( config ) {
|
|
135
|
+
const rawBrokerUrl = config.brokerUrl ?? ENV_VARS.mqttBrokerUrl;
|
|
136
|
+
const brokerUrl = typeof rawBrokerUrl === 'string' ? rawBrokerUrl.trim() : '';
|
|
137
|
+
if ( !brokerUrl ) {
|
|
138
|
+
throw invalidConfig( 'brokerUrl required — set a non-empty string in .emitter() config or MQTT_BROKER_URL env var' );
|
|
139
|
+
}
|
|
140
|
+
// Address policy (ADR-030): `localhost` is refused before the
|
|
141
|
+
// client is created; a name is warned about just before connect.
|
|
142
|
+
const brokerAddress = assertBrokerNotLocalhost( brokerUrl );
|
|
143
|
+
const redactedBrokerUrl = brokerUrl.replace( /\/\/[^@/]*@/, '//***@' );
|
|
144
|
+
return { brokerUrl, brokerAddress, redactedBrokerUrl };
|
|
145
|
+
}; // resolveBrokerUrl()
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Refuses a missing codec or one without `pack()`.
|
|
149
|
+
* @param {Object} codec - config.codec
|
|
150
|
+
*/
|
|
151
|
+
const assertCodec = function ( codec ) {
|
|
152
|
+
if ( !codec ) {
|
|
153
|
+
throw invalidConfig( 'config.codec is required' );
|
|
154
|
+
}
|
|
155
|
+
if ( typeof codec.pack !== 'function' ) {
|
|
156
|
+
throw invalidConfig( 'config.codec must have a pack() function' );
|
|
157
|
+
}
|
|
158
|
+
}; // assertCodec()
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Validates the three notification callbacks and arms them through
|
|
162
|
+
* the shared callback guard.
|
|
163
|
+
*
|
|
164
|
+
* The wrap matters because all three run inside mqtt.js's publish ack
|
|
165
|
+
* chain, where a user throw would land in the client library
|
|
166
|
+
* (ADR-018). Each wrap is null when the callback is absent, so every
|
|
167
|
+
* no-handler path keeps its exact meaning. That includes the
|
|
168
|
+
* deliberate unhandled-rejection escape hatch for an unhandled
|
|
169
|
+
* delivery failure.
|
|
170
|
+
*
|
|
171
|
+
* @param {Object} config - emitter config
|
|
172
|
+
* @returns {{onDeliveryFailure: function|null, onCritical: function|null, onBackpressure: function|null}}
|
|
173
|
+
*/
|
|
174
|
+
const armCallbacks = function ( config ) {
|
|
175
|
+
assertOptionalCallback( config.onDeliveryFailure, 'onDeliveryFailure' );
|
|
176
|
+
assertOptionalCallback( config.onCritical, 'onCritical' );
|
|
177
|
+
assertOptionalCallback( config.onBackpressure, 'onBackpressure' );
|
|
178
|
+
|
|
179
|
+
const reportCallbackFault = function ( severity, name, detail ) {
|
|
180
|
+
logger.error(
|
|
181
|
+
`winkComposer/mqttEmitter: user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
|
|
182
|
+
);
|
|
183
|
+
};
|
|
184
|
+
const onDeliveryFailure = wrapCallback( config.onDeliveryFailure, {
|
|
185
|
+
name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
|
|
186
|
+
} );
|
|
187
|
+
const onCritical = wrapCallback( config.onCritical, {
|
|
188
|
+
name: 'onCritical', severity: 'red', report: reportCallbackFault
|
|
189
|
+
} );
|
|
190
|
+
const onBackpressure = wrapCallback( config.onBackpressure, {
|
|
191
|
+
name: 'onBackpressure', severity: 'yellow', report: reportCallbackFault
|
|
192
|
+
} );
|
|
193
|
+
return { onDeliveryFailure, onCritical, onBackpressure };
|
|
194
|
+
}; // armCallbacks()
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Resolves the first-connack grace: explicit config → MQTT_CONNECT_GRACE_MS
|
|
198
|
+
* env fallback → 500 ms default (ADR-018 precedence; the fallback lives
|
|
199
|
+
* here, never as a schema sigil). `??` keeps an explicit 0 — "hand the
|
|
200
|
+
* handle back immediately" — from falling through to the env value.
|
|
201
|
+
* Number.isInteger also rejects Infinity, so the wait is bounded by
|
|
202
|
+
* construction.
|
|
203
|
+
*
|
|
204
|
+
* @param {Object} config - emitter config
|
|
205
|
+
* @returns {number} the grace in milliseconds, 0 for no wait
|
|
206
|
+
*/
|
|
207
|
+
const resolveGrace = function ( config ) {
|
|
208
|
+
const connectGraceMs = config.connectGraceMs ?? ENV_VARS.mqttConnectGraceMs;
|
|
209
|
+
if ( !Number.isInteger( connectGraceMs ) || ( connectGraceMs < 0 ) ) {
|
|
210
|
+
throw invalidConfig( 'connectGraceMs must be a non-negative integer (milliseconds); 0 disables the first-connect wait' );
|
|
211
|
+
}
|
|
212
|
+
return connectGraceMs;
|
|
213
|
+
}; // resolveGrace()
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Resolves the unacked cap. Clamped to the 16-bit packet-id space:
|
|
217
|
+
* every unacknowledged QoS-1 message holds a packet id, so one
|
|
218
|
+
* connection can never carry more than 65,535; 60,000 leaves working
|
|
219
|
+
* headroom (same clamp the LevelDB store applied, kept with the same
|
|
220
|
+
* warning).
|
|
221
|
+
*
|
|
222
|
+
* @param {Object} config - emitter config
|
|
223
|
+
* @returns {number} the effective cap
|
|
224
|
+
*/
|
|
225
|
+
const resolveQueueSize = function ( config ) {
|
|
226
|
+
const requestedQueueSize = config.maxQueueSize || DEFAULT_MAX_QUEUE_SIZE;
|
|
227
|
+
if ( requestedQueueSize > MQTT_INFLIGHT_ID_LIMIT ) {
|
|
228
|
+
logger.warn(
|
|
229
|
+
`winkComposer/mqttEmitter: maxQueueSize ${requestedQueueSize} exceeds the MQTT ` +
|
|
230
|
+
`packet-id ceiling — clamped to ${MQTT_INFLIGHT_ID_LIMIT}`
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
return Math.min( requestedQueueSize, MQTT_INFLIGHT_ID_LIMIT );
|
|
234
|
+
}; // resolveQueueSize()
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Builds the mqtt.js connect options.
|
|
238
|
+
*
|
|
239
|
+
* NO outgoingStore: the client runs its default SYNCHRONOUS memory
|
|
240
|
+
* store (ADR-021). mqtt.js erases its packet-id bookkeeping on every
|
|
241
|
+
* connack and rebuilds it from a store snapshot asynchronously; with
|
|
242
|
+
* an asynchronous store, writes still in flight at that instant are
|
|
243
|
+
* invisible to the snapshot and get overwritten by id reuse — real
|
|
244
|
+
* QoS-1 loss, measured scaling with store write latency. The
|
|
245
|
+
* synchronous store makes the gap zero-width (validated 2026-07-09,
|
|
246
|
+
* nine-run matrix incl. reconnects at 45k in flight).
|
|
247
|
+
*
|
|
248
|
+
* messageIdProvider: the client's default provider cycles the 16-bit
|
|
249
|
+
* id space with NO in-use check — at 14 k msg/s it wraps every ~5 s,
|
|
250
|
+
* and an id whose PUBACK never arrived gets reassigned, overwriting
|
|
251
|
+
* the unacked packet's memory-store entry (same key). Measured at
|
|
252
|
+
* about one lost publish per 700,000 under sustained load.
|
|
253
|
+
* UniqueMessageIdProvider never reissues an in-use id.
|
|
254
|
+
*
|
|
255
|
+
* @param {Object} config - emitter config
|
|
256
|
+
* @returns {Object} the options handed to `mqtt.connect`
|
|
257
|
+
*/
|
|
258
|
+
const buildClientOptions = function ( config ) {
|
|
259
|
+
// Generate unique client ID if not provided
|
|
260
|
+
const clientId = config.clientId || `wink-${Date.now()}-${Math.random().toString( 36 ).slice( 2, 9 )}`;
|
|
261
|
+
|
|
262
|
+
const mqttOptions = {
|
|
263
|
+
...MQTT_CONFIG,
|
|
264
|
+
clientId,
|
|
265
|
+
// The configured period plus a random share of up to 20%,
|
|
266
|
+
// drawn once here, so a fleet that lost one broker does not
|
|
267
|
+
// retry in step. The configured period is the floor.
|
|
268
|
+
reconnectPeriod: jitteredPeriod( MQTT_CONFIG.reconnectPeriod ),
|
|
269
|
+
messageIdProvider: new mqtt.UniqueMessageIdProvider()
|
|
270
|
+
};
|
|
271
|
+
|
|
272
|
+
// Only add will if explicitly provided (validated + encoded by the
|
|
273
|
+
// helper above; classified throws per ADR-018)
|
|
274
|
+
if ( config.will ) {
|
|
275
|
+
mqttOptions.will = buildWillOptions( config.will, config.codec );
|
|
276
|
+
}
|
|
277
|
+
return mqttOptions;
|
|
278
|
+
}; // buildClientOptions()
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Resolves the whole configuration in the factory's fixed order.
|
|
282
|
+
* Every refusal is a classified `INVALID_CONFIG` throw, and no check
|
|
283
|
+
* has a side effect beyond the clamp warning.
|
|
284
|
+
*
|
|
285
|
+
* @param {Object} config - emitter config (see configSchema in index.js)
|
|
286
|
+
* @returns {Object} the resolved values the factory hands to its modules
|
|
287
|
+
*/
|
|
288
|
+
export const resolveConfig = function ( config ) {
|
|
289
|
+
const { brokerUrl, brokerAddress, redactedBrokerUrl } = resolveBrokerUrl( config );
|
|
290
|
+
assertCodec( config.codec );
|
|
291
|
+
const { onDeliveryFailure, onCritical, onBackpressure } = armCallbacks( config );
|
|
292
|
+
const connectGraceMs = resolveGrace( config );
|
|
293
|
+
const maxQueueSize = resolveQueueSize( config );
|
|
294
|
+
const mqttOptions = buildClientOptions( config );
|
|
295
|
+
|
|
296
|
+
return {
|
|
297
|
+
brokerUrl,
|
|
298
|
+
brokerAddress,
|
|
299
|
+
redactedBrokerUrl,
|
|
300
|
+
codec: config.codec,
|
|
301
|
+
onDeliveryFailure,
|
|
302
|
+
onCritical,
|
|
303
|
+
onBackpressure,
|
|
304
|
+
connectGraceMs,
|
|
305
|
+
maxQueueSize,
|
|
306
|
+
mqttOptions,
|
|
307
|
+
// Injectable for testing (ADR-018 module surface keeps this
|
|
308
|
+
// out of the schema's documented options).
|
|
309
|
+
connect: config.mqttConnectFn || mqtt.connect,
|
|
310
|
+
debug: Boolean( config.debug )
|
|
311
|
+
};
|
|
312
|
+
}; // resolveConfig()
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
// core/emitter-manager/mqtt/shutdown-drain.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The drain of the MQTT emitter: `flush()` and the
|
|
5
|
+
* drain-then-close `shutdown()` (ADR-018 §6 and §7, ADR-021).
|
|
6
|
+
*
|
|
7
|
+
* A clean shutdown resolve is a delivery statement: every accepted
|
|
8
|
+
* message reached a settled outcome before the client closed. The
|
|
9
|
+
* drain is a wait on the emitter's own unacked counter, because
|
|
10
|
+
* mqtt.js's own `end()` does not reliably drain in-flight publishes
|
|
11
|
+
* (a sustained-load finding, kept through the ADR-021 rework).
|
|
12
|
+
*
|
|
13
|
+
* `flush()` runs the same wait and nothing else. It closes nothing and
|
|
14
|
+
* refuses no new work. It resolves when the counter reaches zero
|
|
15
|
+
* within its budget. When the budget passes first it rejects with
|
|
16
|
+
* `DELIVERY_FAILED` and `pending: { count }`, the code QuestDB's
|
|
17
|
+
* flush uses when its own deadline passes. The messages stay in
|
|
18
|
+
* flight; a later acknowledgment still settles them.
|
|
19
|
+
*
|
|
20
|
+
* The shutdown outcome is latched on the first call. Every later
|
|
21
|
+
* caller receives the same promise, so a second call can never
|
|
22
|
+
* contradict the first with an instant clean resolve.
|
|
23
|
+
*
|
|
24
|
+
* The budget reads a monotonic clock. A wall-clock correction during
|
|
25
|
+
* a drain would otherwise shorten or lengthen the wait by the size of
|
|
26
|
+
* the step.
|
|
27
|
+
*
|
|
28
|
+
* How the close detaches (ADR-018 §7). A graceful `end( false )` waits
|
|
29
|
+
* for the client's outgoing map to empty, which a lossy drain cannot
|
|
30
|
+
* deliver. So when rows remain, or the link is down, the close is
|
|
31
|
+
* forced at once: the library destroys the stream and the keepalive
|
|
32
|
+
* timer. A forced close sends no DISCONNECT packet. With a clean
|
|
33
|
+
* session and an in-memory store, nothing is lost that the drain had
|
|
34
|
+
* not already lost. When the drain finished on a live link, the close
|
|
35
|
+
* is graceful, and a timer bounds it. That timer destroys the stream
|
|
36
|
+
* directly, the call the library's own connack timeout makes, because
|
|
37
|
+
* a second `end()` returns at once behind the library's
|
|
38
|
+
* `disconnecting` flag (mqtt.js 5.15.1, `client.js:731-734`).
|
|
39
|
+
*
|
|
40
|
+
* @see ADR-018
|
|
41
|
+
* @see ADR-021
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import { performance } from 'node:perf_hooks';
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The graceful close, bounded by a timer. The library sends DISCONNECT
|
|
48
|
+
* and half-closes the socket. A broker that never answers leaves the
|
|
49
|
+
* socket half-open for ever, so at the deadline the timer destroys the
|
|
50
|
+
* stream itself. The timer runs outside every promise, so a throw
|
|
51
|
+
* there would end the process; it is counted instead.
|
|
52
|
+
*
|
|
53
|
+
* @param {Object} client - mqtt.js client
|
|
54
|
+
* @param {Object} state - The emitter's core state, for the error count
|
|
55
|
+
* @param {number} closeBudget - Milliseconds the broker gets to answer
|
|
56
|
+
* @returns {Promise<void>} resolves when the client closed or the timer fired
|
|
57
|
+
*/
|
|
58
|
+
const closeGracefully = function ( client, state, closeBudget ) {
|
|
59
|
+
return new Promise( ( resolve ) => {
|
|
60
|
+
const forceTimer = setTimeout( () => {
|
|
61
|
+
try {
|
|
62
|
+
client.stream.destroy();
|
|
63
|
+
} catch {
|
|
64
|
+
state.stats.errors += 1;
|
|
65
|
+
}
|
|
66
|
+
resolve();
|
|
67
|
+
}, closeBudget );
|
|
68
|
+
|
|
69
|
+
client.end( false, {}, () => {
|
|
70
|
+
clearTimeout( forceTimer );
|
|
71
|
+
resolve();
|
|
72
|
+
} );
|
|
73
|
+
} );
|
|
74
|
+
}; // closeGracefully()
|
|
75
|
+
|
|
76
|
+
/** The drain budget when the caller gives none, in milliseconds. */
|
|
77
|
+
const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5000;
|
|
78
|
+
|
|
79
|
+
/** One cheap counter read per tick of the drain wait. */
|
|
80
|
+
const DRAIN_POLL_INTERVAL_MS = 25;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* A non-finite or non-positive timeout collapses the drain (Infinity
|
|
84
|
+
* overflows setTimeout, which clamps it to ~1 ms; NaN fails every
|
|
85
|
+
* deadline comparison), so it falls back to the default budget.
|
|
86
|
+
*
|
|
87
|
+
* @param {*} timeout - The caller's timeout
|
|
88
|
+
* @returns {number} a positive finite budget in milliseconds
|
|
89
|
+
*/
|
|
90
|
+
const resolveBudget = function ( timeout ) {
|
|
91
|
+
return ( Number.isFinite( timeout ) && timeout > 0 ) ? timeout : DEFAULT_SHUTDOWN_TIMEOUT_MS;
|
|
92
|
+
}; // resolveBudget()
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Builds `flush` and the latched `shutdown` over the shared state.
|
|
96
|
+
*
|
|
97
|
+
* @param {Object} deps
|
|
98
|
+
* @param {Object} deps.client - mqtt.js client
|
|
99
|
+
* @param {Object} deps.state - The emitter's core state
|
|
100
|
+
* @returns {{flush: function, shutdown: function}}
|
|
101
|
+
*/
|
|
102
|
+
export const createShutdown = function ( { client, state } ) {
|
|
103
|
+
// Shutdown outcome, latched on the first call. A second
|
|
104
|
+
// caller used to get an instant clean resolve — mid-drain, or even
|
|
105
|
+
// after a shutdown that recorded dropped messages. Every caller now
|
|
106
|
+
// receives the first call's promise.
|
|
107
|
+
let shutdownPromise = null;
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Waits for the unacked counter to reach zero, up to the budget.
|
|
111
|
+
* The deadline reads a monotonic clock, so a wall-clock step cannot
|
|
112
|
+
* move it. Sequential awaits are the wait-for-condition pattern.
|
|
113
|
+
*
|
|
114
|
+
* @param {number} budget - Milliseconds to wait, positive and finite
|
|
115
|
+
* @returns {Promise<number>} the milliseconds left in the budget, negative when it ran out
|
|
116
|
+
*/
|
|
117
|
+
const waitForDrain = async function ( budget ) {
|
|
118
|
+
const deadline = performance.now() + budget;
|
|
119
|
+
while ( ( performance.now() < deadline ) && ( state.unacked > 0 ) ) {
|
|
120
|
+
// eslint-disable-next-line no-await-in-loop
|
|
121
|
+
await new Promise( ( r ) => setTimeout( r, DRAIN_POLL_INTERVAL_MS ) );
|
|
122
|
+
}
|
|
123
|
+
return deadline - performance.now();
|
|
124
|
+
}; // waitForDrain()
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Waits for every accepted message to settle, without closing the
|
|
128
|
+
* client or refusing new work (ADR-018 §6).
|
|
129
|
+
*
|
|
130
|
+
* @param {{timeout?: number}} [options] - The budget, default 5000 ms
|
|
131
|
+
* @returns {Promise<void>} resolves when nothing is unacknowledged
|
|
132
|
+
* @throws {Error} DELIVERY_FAILED with `pending: { count }` when the budget passes first
|
|
133
|
+
*/
|
|
134
|
+
const flush = async function ( { timeout = DEFAULT_SHUTDOWN_TIMEOUT_MS } = {} ) {
|
|
135
|
+
const budget = resolveBudget( timeout );
|
|
136
|
+
await waitForDrain( budget );
|
|
137
|
+
if ( state.unacked > 0 ) {
|
|
138
|
+
const err = new Error(
|
|
139
|
+
`winkComposer/mqttEmitter: flush ended with ${state.unacked} message(s) unacknowledged [DELIVERY_FAILED]: no acknowledgment within ${budget} ms`
|
|
140
|
+
);
|
|
141
|
+
err.code = 'DELIVERY_FAILED';
|
|
142
|
+
err.pending = { count: state.unacked };
|
|
143
|
+
throw err;
|
|
144
|
+
}
|
|
145
|
+
}; // flush()
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* The real shutdown body. `shutdown` below latches its promise so
|
|
149
|
+
* every caller receives this one outcome.
|
|
150
|
+
*
|
|
151
|
+
* **Drain-then-close** (a sustained-load finding, kept through the
|
|
152
|
+
* ADR-021 rework): mqtt.js's own `end()` does not reliably drain
|
|
153
|
+
* in-flight publishes, so the emitter waits on its own counter.
|
|
154
|
+
*
|
|
155
|
+
* 1. Sets `state.shuttingDown` so `publishNow` rejects new work
|
|
156
|
+
* (returns `SHUTTING_DOWN` immediately).
|
|
157
|
+
* 2. **Polls the unacked counter until zero**, using the whole
|
|
158
|
+
* `timeout` budget — no early give-up. The wait runs whether or
|
|
159
|
+
* not the connection is up at the moment of the call. With the
|
|
160
|
+
* default 5 s budget and 5 s reconnect period, at most one
|
|
161
|
+
* reconnect attempt can land inside the drain, and a connect
|
|
162
|
+
* that hangs cannot complete within it (the connect timeout is
|
|
163
|
+
* 30 s).
|
|
164
|
+
* 3. Once drained on a live link, calls `client.end( false )` with
|
|
165
|
+
* the remaining time as the broker-DISCONNECT budget. With rows
|
|
166
|
+
* still unacknowledged, or the link down, calls
|
|
167
|
+
* `client.end( true )` at once instead.
|
|
168
|
+
* 4. If the broker hangs the DISCONNECT, the timer destroys the
|
|
169
|
+
* stream directly (see the file header).
|
|
170
|
+
*
|
|
171
|
+
* The old design's shutdown re-drive (re-sending stragglers through
|
|
172
|
+
* a store stream scan) died with the disk store: with the memory
|
|
173
|
+
* store there is no store to scan, and a message whose PUBACK was
|
|
174
|
+
* lost in-session is re-sent by the client itself on the next
|
|
175
|
+
* reconnect. If it is still unacknowledged at the deadline, the
|
|
176
|
+
* loss report counts it.
|
|
177
|
+
*
|
|
178
|
+
* A clean resolve is a delivery statement (ADR-018). When the
|
|
179
|
+
* drain could not complete, shutdown rejects with classified
|
|
180
|
+
* `SHUTDOWN_TIMEOUT` and `dropped: { count }` — the counter's exact
|
|
181
|
+
* value — after the close has already happened (teardown first,
|
|
182
|
+
* then the report). This fires CONNECTED OR NOT: with no disk
|
|
183
|
+
* store, nothing survives the process, so a disconnected shutdown
|
|
184
|
+
* with pending messages is a real loss and is reported as one. (The
|
|
185
|
+
* wal-backed design resolved that case clean on purpose — its disk
|
|
186
|
+
* store held the messages for the next session. ADR-021 traded that
|
|
187
|
+
* away and says so.)
|
|
188
|
+
*/
|
|
189
|
+
const doShutdown = async function ( timeout ) {
|
|
190
|
+
state.shuttingDown = true;
|
|
191
|
+
|
|
192
|
+
const remaining = await waitForDrain( resolveBudget( timeout ) );
|
|
193
|
+
|
|
194
|
+
// Whatever is still unacknowledged goes undelivered — publishNow
|
|
195
|
+
// stopped accepting work at step 1, so the counter has settled
|
|
196
|
+
// into its final value. It becomes the `dropped` payload below.
|
|
197
|
+
const undeliveredCount = state.unacked;
|
|
198
|
+
const closeBudget = remaining > 0 ? remaining : 1;
|
|
199
|
+
|
|
200
|
+
if ( ( undeliveredCount > 0 ) || !state.connected ) {
|
|
201
|
+
// Forced: the library destroys the stream and the keepalive
|
|
202
|
+
// at once, and the call returns synchronously.
|
|
203
|
+
client.end( true );
|
|
204
|
+
} else {
|
|
205
|
+
await closeGracefully( client, state, closeBudget );
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Teardown first, then the report: the close has already happened;
|
|
209
|
+
// an incomplete drain must not read as a clean one (ADR-018).
|
|
210
|
+
if ( undeliveredCount > 0 ) {
|
|
211
|
+
const err = new Error(
|
|
212
|
+
`winkComposer/mqttEmitter: shutdown closed with ${undeliveredCount} message(s) unacknowledged`
|
|
213
|
+
);
|
|
214
|
+
err.code = 'SHUTDOWN_TIMEOUT';
|
|
215
|
+
err.dropped = { count: undeliveredCount };
|
|
216
|
+
throw err;
|
|
217
|
+
}
|
|
218
|
+
return undefined;
|
|
219
|
+
}; // doShutdown()
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Graceful shutdown — drain semantics with a timeout floor.
|
|
223
|
+
*
|
|
224
|
+
* Accepts the ADR-018 shutdown-contract shape `{ timeout }` (defaults to
|
|
225
|
+
* 5000ms). The destructure with `= {}` lets callers invoke
|
|
226
|
+
* `shutdown()` with no argument, `shutdown( {} )`, or
|
|
227
|
+
* `shutdown( { timeout: N } )`.
|
|
228
|
+
*
|
|
229
|
+
* The outcome is latched: the first call runs the shutdown, every
|
|
230
|
+
* later call returns the same promise. A caller
|
|
231
|
+
* arriving mid-drain waits for the real teardown; a caller arriving
|
|
232
|
+
* after a lossy shutdown sees the same classified rejection, never
|
|
233
|
+
* a contradicting clean resolve. One consequence: the first
|
|
234
|
+
* caller's `{ timeout }` governs; a later caller's is ignored.
|
|
235
|
+
*
|
|
236
|
+
* @param {{timeout?: number}} [options]
|
|
237
|
+
* @returns {Promise<void>}
|
|
238
|
+
*/
|
|
239
|
+
const shutdown = function ( { timeout = DEFAULT_SHUTDOWN_TIMEOUT_MS } = {} ) {
|
|
240
|
+
if ( !shutdownPromise ) {
|
|
241
|
+
shutdownPromise = doShutdown( timeout );
|
|
242
|
+
}
|
|
243
|
+
return shutdownPromise;
|
|
244
|
+
}; // shutdown()
|
|
245
|
+
|
|
246
|
+
return { flush, shutdown };
|
|
247
|
+
}; // createShutdown()
|
|
@@ -84,7 +84,7 @@ const assertColumnFacts = function ( columns ) {
|
|
|
84
84
|
!Number.isFinite( colSpec.resolution ) ||
|
|
85
85
|
colSpec.resolution <= 0 ) {
|
|
86
86
|
const err = new Error(
|
|
87
|
-
`
|
|
87
|
+
`winkComposer/terminal: column '${colName}' has type 'float64' ` +
|
|
88
88
|
`but resolution is '${colSpec.resolution}'; ` +
|
|
89
89
|
'expected positive finite number (or omit the field to use the global precision)'
|
|
90
90
|
);
|
|
@@ -21,6 +21,9 @@
|
|
|
21
21
|
* - `getPressure()` returns `0` — Terminal has no observable buffer (stdout drains
|
|
22
22
|
* to the kernel); a trivial sink meets the pressure obligation with a
|
|
23
23
|
* constant 0 (ADR-018).
|
|
24
|
+
* - `flush()` resolves at once, for the same reason. It is on the handle
|
|
25
|
+
* because the wiring layer requires the whole sink floor of every
|
|
26
|
+
* emitter (ADR-018 §6).
|
|
24
27
|
* - `getHealth()` returns `{ status: 'green', connected: true, pressure: 0 }` —
|
|
25
28
|
* Terminal is always healthy and always "connected" to stdout (there is no
|
|
26
29
|
* connection to lose). Uniform semantics with MQTT emitter / QuestDB.
|
|
@@ -237,7 +240,7 @@ const formatVerbose = function ( msg, formatValue ) {
|
|
|
237
240
|
* the global `precision` config. Without an asset class entirely,
|
|
238
241
|
* every column falls back — the original global-precision behaviour,
|
|
239
242
|
* unchanged.
|
|
240
|
-
* @returns {Object} Emitter instance with publishNow, getPressure, getHealth, shutdown
|
|
243
|
+
* @returns {Object} Emitter instance with publishNow, flush, getPressure, getHealth, shutdown
|
|
241
244
|
*/
|
|
242
245
|
export const createEmitter = function ( config = {} ) {
|
|
243
246
|
const {
|
|
@@ -281,15 +284,15 @@ export const createEmitter = function ( config = {} ) {
|
|
|
281
284
|
const ts = new Date().toLocaleTimeString();
|
|
282
285
|
const header = `${linePrefix}── ${ts} ── ${topic} ──`;
|
|
283
286
|
|
|
284
|
-
console.log( header );
|
|
287
|
+
console.log( header ); // eslint-disable-line no-console -- product output, not diagnostics
|
|
285
288
|
|
|
286
289
|
if ( verbose ) {
|
|
287
|
-
console.log( formatVerbose( msg, formatValue ) );
|
|
290
|
+
console.log( formatVerbose( msg, formatValue ) ); // eslint-disable-line no-console -- product output, not diagnostics
|
|
288
291
|
} else {
|
|
289
|
-
console.log( formatCompact( msg, formatValue ) );
|
|
292
|
+
console.log( formatCompact( msg, formatValue ) ); // eslint-disable-line no-console -- product output, not diagnostics
|
|
290
293
|
}
|
|
291
294
|
|
|
292
|
-
console.log(); //
|
|
295
|
+
console.log(); // eslint-disable-line no-console -- product output: blank line for readability
|
|
293
296
|
|
|
294
297
|
return RESULT_OK;
|
|
295
298
|
},
|
|
@@ -305,6 +308,18 @@ export const createEmitter = function ( config = {} ) {
|
|
|
305
308
|
return 0;
|
|
306
309
|
},
|
|
307
310
|
|
|
311
|
+
/**
|
|
312
|
+
* No-op flush (the sink floor, ADR-018 §6). Terminal has nothing
|
|
313
|
+
* buffered to wait for: stdout drains to the kernel without our
|
|
314
|
+
* help. The unused option keeps the same shape as `shutdown`.
|
|
315
|
+
*
|
|
316
|
+
* @param {{timeout?: number}} [_options]
|
|
317
|
+
* @returns {Promise<void>}
|
|
318
|
+
*/
|
|
319
|
+
flush: async function ( { timeout: _timeout = 0 } = {} ) {
|
|
320
|
+
// Nothing to wait for.
|
|
321
|
+
},
|
|
322
|
+
|
|
308
323
|
/**
|
|
309
324
|
* Health snapshot for operator monitoring. Terminal is always healthy
|
|
310
325
|
* and always "connected" to stdout — there is no transport state to
|