@winkjs/composer 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +2 -2
- package/src/composer.js +1 -1
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +103 -658
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/index.js +16 -1
- package/src/core/env-vars.js +92 -24
- package/src/core/logger/index.js +1 -1
- package/src/core/source-manager/mqtt/client.js +197 -63
- package/src/core/source-manager/mqtt/dedup.js +16 -11
- package/src/core/source-manager/mqtt/index.js +32 -5
- package/src/core/source-manager/mqtt/status.js +168 -56
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +542 -862
- package/src/core/storage-manager/questdb/persist-plan.js +128 -167
- package/src/core/storage-manager/questdb/resolve-options.js +266 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +23 -2
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/wiring/wire-emitters.js +3 -1
- package/src/core/wiring/wire-storages.js +10 -2
- package/src/flow/flow.js +24 -3
- package/src/flow/run.js +24 -2
package/src/core/env-vars.js
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
import os from 'os';
|
|
6
6
|
|
|
7
|
+
import { classifyAddress, localhostRefusalDetail } from './utils/address/index.js';
|
|
8
|
+
|
|
7
9
|
// ============================================================================
|
|
8
10
|
// ENVIRONMENT VARIABLE DEFINITIONS
|
|
9
11
|
// ============================================================================
|
|
@@ -63,7 +65,7 @@ const ENV_VARS = {
|
|
|
63
65
|
// `processMessage` (file replays, testHarness, the headless driver):
|
|
64
66
|
// for them the yield tick is the only planned chance for sink flush
|
|
65
67
|
// timers and socket I/O to run. 500 ms is half the tightest background
|
|
66
|
-
// timer it must not starve (
|
|
68
|
+
// timer it must not starve (QUESTDB_FLUSH_INTERVAL_MS, default 1000).
|
|
67
69
|
// Per-flow override: `.yield( { threshold } )`. Infinity = never yield.
|
|
68
70
|
yieldTimeThresholdMs: rawYieldMs === '' ? NaN : Number( rawYieldMs ),
|
|
69
71
|
|
|
@@ -89,22 +91,45 @@ const ENV_VARS = {
|
|
|
89
91
|
mqttSourceDedupWindowMs: parseInt( process.env.MQTT_SOURCE_DEDUP_WINDOW_MS ?? '120000', 10 ),
|
|
90
92
|
mqttSourceDedupMaxEntries: parseInt( process.env.MQTT_SOURCE_DEDUP_MAX_ENTRIES ?? '65536', 10 ),
|
|
91
93
|
|
|
92
|
-
// QuestDB Configuration
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
94
|
+
// QuestDB Configuration. The address defaults are loopback literals,
|
|
95
|
+
// never the name `localhost` (ADR-030). A name can resolve to two
|
|
96
|
+
// addresses, `::1` and `127.0.0.1`, and QuestDB may listen on only
|
|
97
|
+
// one; the run-5 soak lost its write path for four hours that way.
|
|
98
|
+
// The `hostPort` and `mqttUrl` validators below refuse `localhost`
|
|
99
|
+
// outright, so a deployment that sets it stops here, at import.
|
|
100
|
+
questdbIlpUrl: ( process.env.QUESTDB_ILP_URL ?? '127.0.0.1:9000' ).trim(),
|
|
101
|
+
questdbPgUrl: ( process.env.QUESTDB_PG_URL ?? '127.0.0.1:8812' ).trim(),
|
|
98
102
|
// Optional QuestDB settings (undefined if not set)
|
|
99
|
-
questdbAutoFlushRows: process.env.QUESTDB_AUTO_FLUSH_ROWS ?
|
|
100
|
-
parseInt( process.env.QUESTDB_AUTO_FLUSH_ROWS, 10 ) : undefined,
|
|
101
|
-
questdbAutoFlushIntervalMs: process.env.QUESTDB_AUTO_FLUSH_INTERVAL_MS ?
|
|
102
|
-
parseInt( process.env.QUESTDB_AUTO_FLUSH_INTERVAL_MS, 10 ) : undefined,
|
|
103
103
|
questdbMaxBufSize: process.env.QUESTDB_MAX_BUF_SIZE ?
|
|
104
104
|
parseInt( process.env.QUESTDB_MAX_BUF_SIZE, 10 ) : undefined,
|
|
105
105
|
questdbRetryTimeout: process.env.QUESTDB_RETRY_TIMEOUT ?
|
|
106
106
|
parseInt( process.env.QUESTDB_RETRY_TIMEOUT, 10 ) : undefined,
|
|
107
107
|
|
|
108
|
+
// QuestDB flush settings owned by composer (ADR-029). Each carries a
|
|
109
|
+
// value only when the operator set one. The adapter's option
|
|
110
|
+
// resolver supplies the defaults and derives the ceiling and the
|
|
111
|
+
// deadline from the values that won, so a fixed default here would
|
|
112
|
+
// fight a threshold raised in the flow's config.
|
|
113
|
+
questdbFlushRows: process.env.QUESTDB_FLUSH_ROWS ?
|
|
114
|
+
parseInt( process.env.QUESTDB_FLUSH_ROWS, 10 ) : undefined,
|
|
115
|
+
questdbFlushIntervalMs: process.env.QUESTDB_FLUSH_INTERVAL_MS ?
|
|
116
|
+
parseInt( process.env.QUESTDB_FLUSH_INTERVAL_MS, 10 ) : undefined,
|
|
117
|
+
questdbBufferCeilingRows: process.env.QUESTDB_BUFFER_CEILING_ROWS ?
|
|
118
|
+
parseInt( process.env.QUESTDB_BUFFER_CEILING_ROWS, 10 ) : undefined,
|
|
119
|
+
questdbFlushDeadlineMs: process.env.QUESTDB_FLUSH_DEADLINE_MS ?
|
|
120
|
+
parseInt( process.env.QUESTDB_FLUSH_DEADLINE_MS, 10 ) : undefined,
|
|
121
|
+
|
|
122
|
+
// QuestDB transport settings (ADR-029). QUESTDB_STDLIB_HTTP takes
|
|
123
|
+
// the client's own words for `stdlib_http`, `on` or `off`. The
|
|
124
|
+
// adapter's option resolver maps them to a boolean and supplies the
|
|
125
|
+
// default, so each field here carries a value only when set.
|
|
126
|
+
questdbStdlibHttp: process.env.QUESTDB_STDLIB_HTTP ?
|
|
127
|
+
process.env.QUESTDB_STDLIB_HTTP.trim() : undefined,
|
|
128
|
+
questdbRequestTimeout: process.env.QUESTDB_REQUEST_TIMEOUT ?
|
|
129
|
+
parseInt( process.env.QUESTDB_REQUEST_TIMEOUT, 10 ) : undefined,
|
|
130
|
+
questdbInitBufSize: process.env.QUESTDB_INIT_BUF_SIZE ?
|
|
131
|
+
parseInt( process.env.QUESTDB_INIT_BUF_SIZE, 10 ) : undefined,
|
|
132
|
+
|
|
108
133
|
// QuestDB Credentials
|
|
109
134
|
questdbDatabase: ( process.env.QUESTDB_DATABASE ?? 'qdb' ).trim(),
|
|
110
135
|
questdbUser: ( process.env.QUESTDB_USER ?? 'admin' ).trim(),
|
|
@@ -161,20 +186,27 @@ const validators = {
|
|
|
161
186
|
return null;
|
|
162
187
|
},
|
|
163
188
|
|
|
189
|
+
// host:port, with the port required. Accepts a name, an IPv4
|
|
190
|
+
// literal, or a bracketed IPv6 literal (`[::1]:8812`). Refuses
|
|
191
|
+
// `localhost` in every spelling (ADR-030); the shared helper is the
|
|
192
|
+
// one check the adapter schema and factory layers use too.
|
|
164
193
|
hostPort: function ( value ) {
|
|
165
194
|
if ( !value ) return 'Cannot be empty';
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
if ( !hostPortRegex.test( value ) ) {
|
|
195
|
+
const address = classifyAddress( value, 'hostPort' );
|
|
196
|
+
if ( ( address.kind === 'unparsed' ) || ( address.port === undefined ) ) {
|
|
169
197
|
return `Must be host:port format, got: "${value}"`;
|
|
170
198
|
}
|
|
199
|
+
if ( address.kind === 'localhost' ) {
|
|
200
|
+
return localhostRefusalDetail( address );
|
|
201
|
+
}
|
|
171
202
|
return null;
|
|
172
203
|
},
|
|
173
204
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
205
|
+
questdbStdlibHttp: function ( value ) {
|
|
206
|
+
if ( value === undefined ) return null;
|
|
207
|
+
const validWords = [ 'on', 'off' ];
|
|
208
|
+
if ( !validWords.includes( value ) ) {
|
|
209
|
+
return `Must be one of ${validWords.join( ', ' )}, got: "${value}"`;
|
|
178
210
|
}
|
|
179
211
|
return null;
|
|
180
212
|
},
|
|
@@ -208,11 +240,18 @@ const validators = {
|
|
|
208
240
|
return null;
|
|
209
241
|
},
|
|
210
242
|
|
|
243
|
+
// A broker URL. The scheme check comes first; then `localhost` is
|
|
244
|
+
// refused (ADR-030). A URL the parser cannot read is left to the
|
|
245
|
+
// MQTT library, which reports its own error.
|
|
211
246
|
mqttUrl: function ( value ) {
|
|
212
247
|
if ( !value ) return 'Cannot be empty';
|
|
213
248
|
if ( !value.startsWith( 'mqtt://' ) && !value.startsWith( 'mqtts://' ) ) {
|
|
214
249
|
return `Must start with 'mqtt://' or 'mqtts://', got: "${value}"`;
|
|
215
250
|
}
|
|
251
|
+
const address = classifyAddress( value, 'url' );
|
|
252
|
+
if ( address.kind === 'localhost' ) {
|
|
253
|
+
return localhostRefusalDetail( address );
|
|
254
|
+
}
|
|
216
255
|
return null;
|
|
217
256
|
}
|
|
218
257
|
};
|
|
@@ -241,30 +280,59 @@ const validationConfig = [
|
|
|
241
280
|
// QuestDB Configuration
|
|
242
281
|
{ field: 'questdbIlpUrl', validator: validators.hostPort, label: 'QUESTDB_ILP_URL' },
|
|
243
282
|
{ field: 'questdbPgUrl', validator: validators.hostPort, label: 'QUESTDB_PG_URL' },
|
|
244
|
-
{ field: 'questdbFlushMode', validator: validators.questdbFlushMode, label: 'QUESTDB_FLUSH_MODE' },
|
|
245
|
-
{ field: 'questdbIdleFlushAfterMs', validator: validators.nonNegativeInt, originalEnv: 'QUESTDB_IDLE_FLUSH_AFTER_MS' },
|
|
246
|
-
{ field: 'questdbIdleFlushCheckMs', validator: validators.positiveInt, originalEnv: 'QUESTDB_IDLE_FLUSH_CHECK_MS' },
|
|
247
|
-
{ field: 'questdbAutoFlushRows', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_AUTO_FLUSH_ROWS' },
|
|
248
|
-
{ field: 'questdbAutoFlushIntervalMs', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_AUTO_FLUSH_INTERVAL_MS' },
|
|
249
283
|
{ field: 'questdbMaxBufSize', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_MAX_BUF_SIZE' },
|
|
250
284
|
{ field: 'questdbRetryTimeout', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_RETRY_TIMEOUT' },
|
|
285
|
+
{ field: 'questdbFlushRows', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_FLUSH_ROWS' },
|
|
286
|
+
{ field: 'questdbFlushIntervalMs', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_FLUSH_INTERVAL_MS' },
|
|
287
|
+
{ field: 'questdbBufferCeilingRows', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_BUFFER_CEILING_ROWS' },
|
|
288
|
+
{ field: 'questdbFlushDeadlineMs', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_FLUSH_DEADLINE_MS' },
|
|
289
|
+
{ field: 'questdbStdlibHttp', validator: validators.questdbStdlibHttp, label: 'QUESTDB_STDLIB_HTTP' },
|
|
290
|
+
{ field: 'questdbRequestTimeout', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_REQUEST_TIMEOUT' },
|
|
291
|
+
{ field: 'questdbInitBufSize', validator: validators.positiveIntOrUndefined, originalEnv: 'QUESTDB_INIT_BUF_SIZE' },
|
|
251
292
|
// QuestDB Credentials
|
|
252
293
|
{ field: 'questdbDatabase', validator: validators.nonEmptyString, label: 'QUESTDB_DATABASE' },
|
|
253
294
|
{ field: 'questdbUser', validator: validators.nonEmptyString, label: 'QUESTDB_USER' }
|
|
254
295
|
// questdbPassword: no validation — allows empty for passwordless auth
|
|
255
296
|
];
|
|
256
297
|
|
|
298
|
+
// The five QuestDB flush variables 0.7.0 deprecated and 0.8.0 removed
|
|
299
|
+
// (ADR-029 item 10). A deployment that still sets one stops here, at
|
|
300
|
+
// import, and the failure line names what to do instead. Ignoring the
|
|
301
|
+
// variable would let an operator believe a setting took effect.
|
|
302
|
+
const REMOVED_ENV_VARS = [
|
|
303
|
+
{ name: 'QUESTDB_FLUSH_MODE', action: 'delete it, composer owns every flush' },
|
|
304
|
+
{ name: 'QUESTDB_IDLE_FLUSH_AFTER_MS', action: 'delete it, composer owns every flush' },
|
|
305
|
+
{ name: 'QUESTDB_IDLE_FLUSH_CHECK_MS', action: 'use QUESTDB_FLUSH_INTERVAL_MS' },
|
|
306
|
+
{ name: 'QUESTDB_AUTO_FLUSH_ROWS', action: 'use QUESTDB_FLUSH_ROWS' },
|
|
307
|
+
{ name: 'QUESTDB_AUTO_FLUSH_INTERVAL_MS', action: 'delete it, composer owns every flush' }
|
|
308
|
+
];
|
|
309
|
+
|
|
257
310
|
// ============================================================================
|
|
258
311
|
// VALIDATION RUNNER
|
|
259
312
|
// ============================================================================
|
|
260
313
|
const validate = function () {
|
|
261
314
|
const errors = [];
|
|
262
315
|
|
|
316
|
+
// A removed variable is refused by name, even when empty, so a
|
|
317
|
+
// leftover line in an env file is found.
|
|
318
|
+
for ( const removed of REMOVED_ENV_VARS ) {
|
|
319
|
+
if ( process.env[ removed.name ] !== undefined ) {
|
|
320
|
+
errors.push( `${removed.name}: Removed in 0.8.0; ${removed.action}` );
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
263
324
|
// Run each validator
|
|
264
325
|
for ( const config of validationConfig ) {
|
|
265
326
|
const value = ENV_VARS[ config.field ];
|
|
266
327
|
const originalEnv = config.originalEnv ? process.env[ config.originalEnv ] : value;
|
|
267
|
-
|
|
328
|
+
// The label is the variable name as the operator typed it. A row
|
|
329
|
+
// that records the variable uses it; otherwise the name is
|
|
330
|
+
// derived from the field, inserting the underscores BEFORE
|
|
331
|
+
// uppercasing (the other order finds no boundaries and printed
|
|
332
|
+
// QUESTDBRETRYTIMEOUT until 2026-09-05).
|
|
333
|
+
const label = config.label ||
|
|
334
|
+
config.originalEnv ||
|
|
335
|
+
config.field.replace( /([a-z])([A-Z])/g, '$1_$2' ).toUpperCase();
|
|
268
336
|
|
|
269
337
|
const error = config.validator( value, originalEnv || value, label );
|
|
270
338
|
if ( error ) {
|
package/src/core/logger/index.js
CHANGED
|
@@ -33,7 +33,11 @@
|
|
|
33
33
|
* `err.code` vocabulary (per-adapter, documented here per ADR-018):
|
|
34
34
|
* - `INVALID_CONFIG` — setup-time; missing or malformed config
|
|
35
35
|
* field. Thrown synchronously from the factory (ADR-018 fail-fast
|
|
36
|
-
* setup), never emitted.
|
|
36
|
+
* setup), never emitted. Includes a `brokerUrl` whose host is
|
|
37
|
+
* `localhost` (ADR-030): the name can resolve to two addresses and
|
|
38
|
+
* the broker may listen on only one. The message names the literal
|
|
39
|
+
* to set. Refused before the client is created, at the schema (flow
|
|
40
|
+
* definition) and here (direct callers).
|
|
37
41
|
* - `DECODE_ERROR` — runtime, yellow. Two faces: a per-record
|
|
38
42
|
* report for every payload that does not yield a usable record, and
|
|
39
43
|
* a health flip when the decode-error ratio over the last 1,000
|
|
@@ -69,6 +73,15 @@
|
|
|
69
73
|
* - `QUIET_PERIOD_EXCEEDED` — runtime, yellow, opt-in. No packet for
|
|
70
74
|
* longer than the configured `expectedQuietPeriodMs`.
|
|
71
75
|
*
|
|
76
|
+
* Console classification (a token on a log line, not an `err.code`):
|
|
77
|
+
* - `ADDRESS_IS_NAME` — the `brokerUrl` host is a name other
|
|
78
|
+
* than `localhost`. One `logger.warn` line at setup, before the
|
|
79
|
+
* client is created (ADR-030). A name is allowed and the source
|
|
80
|
+
* proceeds; the line tells the operator that only a literal address
|
|
81
|
+
* is immune to a resolver that changes its answer under a running
|
|
82
|
+
* process. This source refuses and warns; it does not probe, because
|
|
83
|
+
* its posture is recovering (ADR-018 §5).
|
|
84
|
+
*
|
|
72
85
|
* Metrics (optional `onMetrics`, ~1 Hz + on transitions): monotonic
|
|
73
86
|
* counters `{delivered, skipped, decodeErrors, reconnects, dedupHits,
|
|
74
87
|
* dedupMisses, dedupBypassed, dedupCacheSize}`. `dedupBypassed` counts
|
|
@@ -107,15 +120,34 @@
|
|
|
107
120
|
* clears. The operator tells them apart, not the code.
|
|
108
121
|
* 4. Safe to kill and re-create: no disk state, no side effects.
|
|
109
122
|
* The cost of a crash is the dedup cache (see dedup.js).
|
|
123
|
+
* 5. No payload size cap and no intake rate cap. A flood of large
|
|
124
|
+
* messages is bounded only by the broker's own per-client limits
|
|
125
|
+
* and by TCP back-pressure on the socket. ADR-026 (proposed) is
|
|
126
|
+
* the home for a cap; the handbook states the limit.
|
|
127
|
+
*
|
|
128
|
+
* Performance, measured on the private benchmark harness (2026-04-24,
|
|
129
|
+
* M4 Max; pre-ADR-022 dedup, whose replacement has the same O(1)
|
|
130
|
+
* per-message profile): composer's own decode/dedup/dispatch path
|
|
131
|
+
* sustains 897 k msg/s at 100 B payloads (317 k at 1 KB). End-to-end
|
|
132
|
+
* through Mosquitto at QoS 1 the single-process ceiling was
|
|
133
|
+
* ~12.9 k msg/s, owned by mqtt.js and the broker round-trip, not this
|
|
134
|
+
* code. At 10 k msg/s steady state every cell ran clean; a
|
|
135
|
+
* 30 M-message run showed no subscriber-side leak.
|
|
110
136
|
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
137
|
+
* Decisions this file follows:
|
|
138
|
+
* - ADR-004 and ADR-013 — the message handler is synchronous and
|
|
139
|
+
* allocates nothing on the success path. The flow's dispatch never
|
|
140
|
+
* waits on it.
|
|
141
|
+
* - ADR-018 — the adapter contract: phases, status shape, codes.
|
|
142
|
+
* - ADR-022 — the dedup cache and its two bounds.
|
|
143
|
+
* - ADR-027 — the callback wrapper scope. `transform` runs under the
|
|
144
|
+
* shared guard. `onMessage` is not wrapped here; the flow's dispatch
|
|
145
|
+
* guard owns it.
|
|
146
|
+
* - ADR-028 — the grammar of every facade line printed here and in
|
|
147
|
+
* status.js.
|
|
148
|
+
* - ADR-030 — `localhost` is refused; a name is warned about.
|
|
149
|
+
* - ADR-026 (proposed) — the external-threat surface. TLS,
|
|
150
|
+
* authentication, and payload caps belong there, not here.
|
|
119
151
|
*
|
|
120
152
|
* @see src/core/source-manager/mqtt/status.js - The reporting rules
|
|
121
153
|
* @see src/core/emitter-manager/mqtt/emitter.js - Emitter counterpart
|
|
@@ -135,7 +167,48 @@ import { createDedupCache } from './dedup.js';
|
|
|
135
167
|
import { createStatusReporter } from './status.js';
|
|
136
168
|
import { isUsableRecord, describeShape } from '../record-shape.js';
|
|
137
169
|
import { wrapTransform, TRANSFORM_THREW } from '../../utils/callback-guard/index.js';
|
|
170
|
+
import { jitteredPeriod } from '../../utils/jitter/index.js';
|
|
138
171
|
import { logger } from '../../logger/index.js';
|
|
172
|
+
import { classifyAddress, localhostRefusalMessage, nameWarningMessage } from '../../utils/address/index.js';
|
|
173
|
+
|
|
174
|
+
// ============================================================================
|
|
175
|
+
// ADDRESS POLICY (ADR-030)
|
|
176
|
+
// ============================================================================
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Classifies the broker address and refuses `localhost`. The schema
|
|
180
|
+
* already refused it at flow definition; this call covers direct
|
|
181
|
+
* callers and carries the classified code. The source has no
|
|
182
|
+
* environment fallback for `brokerUrl`, so no env var is named.
|
|
183
|
+
*
|
|
184
|
+
* @param {string} brokerUrl - The broker URL as configured
|
|
185
|
+
* @returns {Object} The classified address
|
|
186
|
+
* @throws {Error} INVALID_CONFIG when the host is `localhost`
|
|
187
|
+
*/
|
|
188
|
+
const assertBrokerNotLocalhost = function ( brokerUrl ) {
|
|
189
|
+
const address = classifyAddress( brokerUrl, 'url' );
|
|
190
|
+
if ( address.kind === 'localhost' ) {
|
|
191
|
+
const err = new Error(
|
|
192
|
+
`winkComposer/mqttSource: ${localhostRefusalMessage( { field: 'brokerUrl', address } )}`
|
|
193
|
+
);
|
|
194
|
+
err.code = 'INVALID_CONFIG';
|
|
195
|
+
throw err;
|
|
196
|
+
}
|
|
197
|
+
return address;
|
|
198
|
+
}; // assertBrokerNotLocalhost()
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Prints the one ADDRESS_IS_NAME line when the broker host is a name.
|
|
202
|
+
*
|
|
203
|
+
* @param {Object} address - The classified broker address
|
|
204
|
+
*/
|
|
205
|
+
const warnIfBrokerIsName = function ( address ) {
|
|
206
|
+
if ( address.kind === 'name' ) {
|
|
207
|
+
logger.warn(
|
|
208
|
+
`winkComposer/mqttSource: ${nameWarningMessage( { field: 'brokerUrl', host: address.host } )}`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
}; // warnIfBrokerIsName()
|
|
139
212
|
|
|
140
213
|
// ============================================================================
|
|
141
214
|
// CLIENT FACTORY
|
|
@@ -146,7 +219,7 @@ import { logger } from '../../logger/index.js';
|
|
|
146
219
|
* health/metrics reporting.
|
|
147
220
|
*
|
|
148
221
|
* @param {Object} config - Client configuration
|
|
149
|
-
* @param {string} config.brokerUrl - MQTT broker URL (e.g., 'mqtt://
|
|
222
|
+
* @param {string} config.brokerUrl - MQTT broker URL (e.g., 'mqtt://127.0.0.1:1883'; never localhost)
|
|
150
223
|
* @param {string|string[]} config.topics - Topic(s) to subscribe to (supports wildcards)
|
|
151
224
|
* @param {function} config.onMessage - Message handler: (message) => void
|
|
152
225
|
* @param {Object} [config.codec] - Codec for payload decoding (default: JSON.parse)
|
|
@@ -192,6 +265,9 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
192
265
|
err.code = 'INVALID_CONFIG';
|
|
193
266
|
throw err;
|
|
194
267
|
}
|
|
268
|
+
// Address policy (ADR-030): `localhost` is refused before the
|
|
269
|
+
// client is created; a name is warned about just before connect.
|
|
270
|
+
const brokerAddress = assertBrokerNotLocalhost( brokerUrl );
|
|
195
271
|
if ( !topics || ( Array.isArray( topics ) && topics.length === 0 ) ) {
|
|
196
272
|
const err = new Error( 'winkComposer/mqttSource: topics is required' );
|
|
197
273
|
err.code = 'INVALID_CONFIG';
|
|
@@ -232,15 +308,20 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
232
308
|
};
|
|
233
309
|
const guardedTransform = transform ? wrapTransform( transform, reportTransformFault ) : null;
|
|
234
310
|
|
|
235
|
-
//
|
|
236
|
-
|
|
311
|
+
// The auto-generated name carries the start time and a random
|
|
312
|
+
// part, the emitter's shape. Two sources that start in the same
|
|
313
|
+
// millisecond still get different names. That matters because the
|
|
314
|
+
// broker allows one live connection per name. When a second client
|
|
315
|
+
// arrives under a name in use, the broker disconnects the first.
|
|
316
|
+
// Two clients then take the session from each other on every
|
|
317
|
+
// reconnect.
|
|
318
|
+
const generatedClientId = clientId || `wink-source-${Date.now()}-${Math.random().toString( 36 ).slice( 2, 9 )}`;
|
|
237
319
|
|
|
238
320
|
// A persistent session is filed at the broker under the client's
|
|
239
321
|
// name. The auto-generated name changes on every start, so the
|
|
240
|
-
// backlog
|
|
241
|
-
//
|
|
242
|
-
//
|
|
243
|
-
// only — nothing here touches the message path.
|
|
322
|
+
// backlog saved under the previous run's name is never delivered.
|
|
323
|
+
// Warn once at startup. The fix is one config line, on the setup
|
|
324
|
+
// path only.
|
|
244
325
|
if ( !clientId && cleanStart !== true ) {
|
|
245
326
|
logger.warn(
|
|
246
327
|
'winkComposer/mqttSource: no clientId configured — this session ' +
|
|
@@ -251,10 +332,14 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
251
332
|
);
|
|
252
333
|
}
|
|
253
334
|
|
|
254
|
-
// Build MQTT options, allowing cleanStart override
|
|
335
|
+
// Build MQTT options, allowing cleanStart override. The reconnect
|
|
336
|
+
// period is the configured one plus a random share of up to 20%,
|
|
337
|
+
// drawn once here. So a fleet that lost one broker does not retry
|
|
338
|
+
// in step. The configured period is the floor.
|
|
255
339
|
const mqttOptions = {
|
|
256
340
|
...MQTT_SOURCE_CONFIG,
|
|
257
|
-
clientId: generatedClientId
|
|
341
|
+
clientId: generatedClientId,
|
|
342
|
+
reconnectPeriod: jitteredPeriod( MQTT_SOURCE_CONFIG.reconnectPeriod )
|
|
258
343
|
};
|
|
259
344
|
|
|
260
345
|
// Map the user-facing `cleanStart` key (MQTT 5 term) onto the
|
|
@@ -264,7 +349,9 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
264
349
|
mqttOptions.clean = cleanStart;
|
|
265
350
|
}
|
|
266
351
|
|
|
267
|
-
// Create MQTT client
|
|
352
|
+
// Create MQTT client. The name warning goes first, so it prints
|
|
353
|
+
// before any socket opens (ADR-030).
|
|
354
|
+
warnIfBrokerIsName( brokerAddress );
|
|
268
355
|
const client = mqttConnectFn( brokerUrl, mqttOptions );
|
|
269
356
|
|
|
270
357
|
// One heartbeat per second: re-evaluates the time-based health
|
|
@@ -303,7 +390,10 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
303
390
|
const dedupId = userProps ? userProps[ WINK_NAMESPACE.dedupId ] : undefined;
|
|
304
391
|
|
|
305
392
|
// Dedup is opt-in by construction (ADR-022): no id → bypass.
|
|
306
|
-
|
|
393
|
+
// A repeated MQTT 5 user property parses to an array, and an
|
|
394
|
+
// array can never equal an earlier one. So any non-string id
|
|
395
|
+
// bypasses too, counted in dedupBypassed rather than cached.
|
|
396
|
+
if ( typeof dedupId !== 'string' ) {
|
|
307
397
|
reporter.bypassed();
|
|
308
398
|
} else if ( dedup.isDuplicate( dedupId ) ) {
|
|
309
399
|
reporter.dupSkipped();
|
|
@@ -312,16 +402,17 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
312
402
|
reporter.idAccepted();
|
|
313
403
|
}
|
|
314
404
|
|
|
315
|
-
// Decode payload, check its shape, and attach metadata
|
|
316
|
-
// guarded region. A failure anywhere in it is skipped,
|
|
317
|
-
// classified, and reported per record (ADR-018)
|
|
318
|
-
// continues.
|
|
319
|
-
//
|
|
320
|
-
//
|
|
321
|
-
//
|
|
322
|
-
//
|
|
323
|
-
//
|
|
324
|
-
//
|
|
405
|
+
// Decode the payload, check its shape, and attach metadata in
|
|
406
|
+
// one guarded region. A failure anywhere in it is skipped,
|
|
407
|
+
// classified, and reported per record (ADR-018). The stream
|
|
408
|
+
// continues.
|
|
409
|
+
//
|
|
410
|
+
// The shape guard is needed because a valid JSON document can
|
|
411
|
+
// be a scalar or a bare array. The attach is inside the guard
|
|
412
|
+
// because a codec can return a frozen record, or one with a
|
|
413
|
+
// non-writable _topic. The assignment then throws in strict
|
|
414
|
+
// mode. decodeOk() runs only when the whole record survived,
|
|
415
|
+
// so the ring gets exactly one entry per message.
|
|
325
416
|
let message;
|
|
326
417
|
try {
|
|
327
418
|
if ( codec && typeof codec.unpack === 'function' ) {
|
|
@@ -341,7 +432,15 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
341
432
|
|
|
342
433
|
reporter.decodeOk();
|
|
343
434
|
} catch ( err ) {
|
|
344
|
-
|
|
435
|
+
// The parser's own message can echo a fragment of the
|
|
436
|
+
// payload. Payload text is data, so it stays below the
|
|
437
|
+
// warn level (ADR-028). The report names the topic and
|
|
438
|
+
// the size. The parser's reason goes to a debug line,
|
|
439
|
+
// guarded so the hot path builds nothing when debug is off.
|
|
440
|
+
reporter.decodeFailed( `topic '${topic}': payload of ${payload.length} bytes could not be decoded — message skipped` );
|
|
441
|
+
if ( logger.debugOn ) {
|
|
442
|
+
logger.debug( `winkComposer/mqttSource: decode failed [DECODE_ERROR]: topic '${topic}': ${err.message}` );
|
|
443
|
+
}
|
|
345
444
|
return;
|
|
346
445
|
}
|
|
347
446
|
|
|
@@ -349,12 +448,13 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
349
448
|
// at startup. A throw skips this one message with a per-record
|
|
350
449
|
// CALLBACK_FAILED report, and the stream continues. User code
|
|
351
450
|
// must never propagate into mqtt.js's event processing
|
|
352
|
-
// (transform contract, 2026-07-11).
|
|
353
|
-
//
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
//
|
|
357
|
-
//
|
|
451
|
+
// (transform contract, 2026-07-11).
|
|
452
|
+
//
|
|
453
|
+
// The return is held to the same record shape as the payload.
|
|
454
|
+
// A null or undefined return stays the intentional silent
|
|
455
|
+
// drop. A scalar or array return is one per-record
|
|
456
|
+
// CALLBACK_FAILED. The shape check runs only when a transform
|
|
457
|
+
// is configured. The plain path pays nothing.
|
|
358
458
|
let finalMessage;
|
|
359
459
|
if ( guardedTransform ) {
|
|
360
460
|
finalMessage = guardedTransform( message, topic );
|
|
@@ -398,51 +498,85 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
398
498
|
// STOP FUNCTION
|
|
399
499
|
// ========================================================================
|
|
400
500
|
|
|
501
|
+
// The stop outcome is latched on the first call. A second caller
|
|
502
|
+
// receives the same promise, so it can never report a clean stop
|
|
503
|
+
// while the first call is still closing.
|
|
504
|
+
let stopPromise = null;
|
|
505
|
+
|
|
401
506
|
/**
|
|
402
507
|
* Stop the MQTT client, with a time budget.
|
|
403
508
|
*
|
|
404
|
-
* Per ADR-018, source stop functions take a `{ timeout }`.
|
|
405
|
-
*
|
|
406
|
-
* which waits for any messages still being sent. If that does not
|
|
407
|
-
* finish within `timeout` ms, we ask for a forced disconnect
|
|
408
|
-
* (`client.end(true)`), which closes the socket right away — so the
|
|
409
|
-
* rest of the shutdown can proceed even if the broker is slow or
|
|
410
|
-
* unreachable. The forced path is reported with the sources' shared
|
|
411
|
-
* `note` convention.
|
|
509
|
+
* Per ADR-018, source stop functions take a `{ timeout }`. How the
|
|
510
|
+
* close detaches follows the library's own rules (mqtt.js 5.15.1):
|
|
412
511
|
*
|
|
413
|
-
*
|
|
414
|
-
*
|
|
512
|
+
* - Not connected (the first connect still pending, the link down,
|
|
513
|
+
* or the library waiting to retry): `end( true )` at once. The
|
|
514
|
+
* library destroys the stream, clears its connect and reconnect
|
|
515
|
+
* timers, and calls back on the next tick. A graceful
|
|
516
|
+
* `end( false )` would call back at once here too, but it would
|
|
517
|
+
* not destroy the stream (`client.js:921-924`). A pending
|
|
518
|
+
* connect would then hold the process for the whole connect
|
|
519
|
+
* timeout.
|
|
520
|
+
* - Connected: `end( false )` sends DISCONNECT and half-closes the
|
|
521
|
+
* socket, then waits for the broker to close its side. A timer
|
|
522
|
+
* bounds that wait. At the budget the timer destroys the stream
|
|
523
|
+
* directly. That is the call the library's own connect timeout
|
|
524
|
+
* makes. A second `end()` would do nothing: it returns at once
|
|
525
|
+
* behind the library's `disconnecting` flag (`client.js:731-734`).
|
|
526
|
+
* The forced path is reported with the sources' shared `note`
|
|
527
|
+
* convention.
|
|
415
528
|
*
|
|
416
|
-
*
|
|
529
|
+
* The timer is `unref()`ed so it does not keep Node alive while a
|
|
530
|
+
* clean disconnect is in progress. Either path settles the promise
|
|
531
|
+
* once. The default time budget (5000 ms) matches the sinks.
|
|
417
532
|
*
|
|
418
533
|
* @param {Object} [options] - Stop options
|
|
419
|
-
* @param {number} [options.timeout=5000] - Max ms to wait for clean disconnect
|
|
534
|
+
* @param {number} [options.timeout=5000] - Max ms to wait for the clean disconnect
|
|
420
535
|
* @returns {Promise<void>} Resolves once the client is closed (clean or forced)
|
|
421
536
|
*/
|
|
422
537
|
const stop = function ( { timeout = 5000 } = {} ) {
|
|
538
|
+
if ( stopPromise ) {
|
|
539
|
+
return stopPromise;
|
|
540
|
+
}
|
|
423
541
|
clearInterval( cadence );
|
|
424
|
-
|
|
542
|
+
stopPromise = new Promise( function ( resolve ) {
|
|
425
543
|
let settled = false;
|
|
426
|
-
|
|
427
|
-
const onClosed = function () {
|
|
428
|
-
if ( settled ) return;
|
|
429
|
-
settled = true;
|
|
430
|
-
if ( forceTimer ) clearTimeout( forceTimer );
|
|
544
|
+
const reportClean = function () {
|
|
431
545
|
reporter.stopped();
|
|
546
|
+
};
|
|
547
|
+
const reportForced = function () {
|
|
548
|
+
reporter.stopForced( timeout );
|
|
549
|
+
};
|
|
550
|
+
// The library's close callback also fires after the timer's
|
|
551
|
+
// destroy, so the second arrival must settle nothing.
|
|
552
|
+
const settle = function ( report ) {
|
|
553
|
+
if ( settled ) {
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
settled = true;
|
|
557
|
+
report();
|
|
432
558
|
resolve();
|
|
433
559
|
};
|
|
434
|
-
forceTimer = setTimeout( function () {
|
|
435
560
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
561
|
+
if ( !client.connected ) {
|
|
562
|
+
client.end( true, {}, function () {
|
|
563
|
+
settle( reportClean );
|
|
564
|
+
} );
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
const forceTimer = setTimeout( function () {
|
|
569
|
+
client.stream.destroy();
|
|
570
|
+
settle( reportForced );
|
|
440
571
|
}, timeout );
|
|
441
|
-
// Don't keep Node alive on the timer while the broker is
|
|
442
|
-
// closing cleanly. Either path resolves the Promise.
|
|
443
572
|
forceTimer.unref();
|
|
444
|
-
|
|
573
|
+
|
|
574
|
+
client.end( false, {}, function () {
|
|
575
|
+
clearTimeout( forceTimer );
|
|
576
|
+
settle( reportClean );
|
|
577
|
+
} );
|
|
445
578
|
} );
|
|
579
|
+
return stopPromise;
|
|
446
580
|
};
|
|
447
581
|
|
|
448
582
|
// Expose for testing
|