@winkjs/composer 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -3
- package/src/composer.js +1 -1
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +108 -633
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +20 -5
- package/src/core/env-vars.js +143 -16
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +276 -83
- package/src/core/source-manager/mqtt/dedup.js +19 -14
- package/src/core/source-manager/mqtt/index.js +33 -6
- package/src/core/source-manager/mqtt/status.js +211 -46
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +592 -827
- package/src/core/storage-manager/questdb/persist-plan.js +147 -162
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +176 -0
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +7 -4
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +14 -5
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +62 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +200 -40
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
* Message flow:
|
|
16
16
|
* MQTT broker → subscribe → extract userProperties → check dedup cache
|
|
17
17
|
* → if duplicate: skip, count (dedupHits — not a status event)
|
|
18
|
-
* → if new: decode payload →
|
|
18
|
+
* → if new: decode payload → shape guard + metadata attach
|
|
19
|
+
* → transform (optional) → onMessage(msg)
|
|
19
20
|
*
|
|
20
21
|
* Status shape — the ADR-018 core, plus this adapter's additions:
|
|
21
22
|
* `{status: 'green' | 'yellow' | 'red', connected, phase,
|
|
@@ -32,16 +33,33 @@
|
|
|
32
33
|
* `err.code` vocabulary (per-adapter, documented here per ADR-018):
|
|
33
34
|
* - `INVALID_CONFIG` — setup-time; missing or malformed config
|
|
34
35
|
* field. Thrown synchronously from the factory (ADR-018 fail-fast
|
|
35
|
-
* setup), never emitted.
|
|
36
|
+
* setup), never emitted. Includes a `brokerUrl` whose host is
|
|
37
|
+
* `localhost` (ADR-030): the name can resolve to two addresses and
|
|
38
|
+
* the broker may listen on only one. The message names the literal
|
|
39
|
+
* to set. Refused before the client is created, at the schema (flow
|
|
40
|
+
* definition) and here (direct callers).
|
|
36
41
|
* - `DECODE_ERROR` — runtime, yellow. Two faces: a per-record
|
|
37
|
-
* report for every payload that
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
42
|
+
* report for every payload that does not yield a usable record, and
|
|
43
|
+
* a health flip when the decode-error ratio over the last 1,000
|
|
44
|
+
* messages exceeds 1 %. The per-record face covers more than parse
|
|
45
|
+
* failures (a widening of ADR-018 §9's original wording): a payload
|
|
46
|
+
* that decodes to a scalar, null, or a bare array is rejected too,
|
|
47
|
+
* and so is a record the metadata attach cannot write to (a frozen
|
|
48
|
+
* record from a custom codec). Skip, classify, continue — the
|
|
49
|
+
* report names what arrived. A bare scalar landing here usually
|
|
50
|
+
* means a too-wide topic subscription; narrow the topic filter.
|
|
51
|
+
* - `CALLBACK_FAILED` — runtime, yellow. A piece of user code
|
|
52
|
+
* failed. It covers two cases. Case one: the user's `transform`
|
|
53
|
+
* threw, or returned a scalar or array where a record object was
|
|
54
|
+
* needed. That one message is skipped (counted in `skipped`) and
|
|
55
|
+
* the stream continues. Fix the transform function — the report
|
|
56
|
+
* names the topic and the fault. A null/undefined return is NOT
|
|
57
|
+
* this case: it is the documented intentional drop. Uniform with
|
|
58
|
+
* the CSV source (transform contract, 2026-07-11). Case two: the
|
|
59
|
+
* user's `onStatus` or `onMetrics` itself threw or rejected. The
|
|
60
|
+
* shared callback guard contains that fault (ADR-018, wired in
|
|
61
|
+
* status.js). The stream continues, and each fault is reported
|
|
62
|
+
* once with the detail.
|
|
45
63
|
* - `SUBSCRIBE_FAILED` — runtime, red. The broker refused the
|
|
46
64
|
* subscription (typically ACL). Red immediately: nothing retries a
|
|
47
65
|
* subscribe until the next reconnect, so a deaf-but-connected source
|
|
@@ -55,6 +73,15 @@
|
|
|
55
73
|
* - `QUIET_PERIOD_EXCEEDED` — runtime, yellow, opt-in. No packet for
|
|
56
74
|
* longer than the configured `expectedQuietPeriodMs`.
|
|
57
75
|
*
|
|
76
|
+
* Console classification (a token on a log line, not an `err.code`):
|
|
77
|
+
* - `ADDRESS_IS_NAME` — the `brokerUrl` host is a name other
|
|
78
|
+
* than `localhost`. One `logger.warn` line at setup, before the
|
|
79
|
+
* client is created (ADR-030). A name is allowed and the source
|
|
80
|
+
* proceeds; the line tells the operator that only a literal address
|
|
81
|
+
* is immune to a resolver that changes its answer under a running
|
|
82
|
+
* process. This source refuses and warns; it does not probe, because
|
|
83
|
+
* its posture is recovering (ADR-018 §5).
|
|
84
|
+
*
|
|
58
85
|
* Metrics (optional `onMetrics`, ~1 Hz + on transitions): monotonic
|
|
59
86
|
* counters `{delivered, skipped, decodeErrors, reconnects, dedupHits,
|
|
60
87
|
* dedupMisses, dedupBypassed, dedupCacheSize}`. `dedupBypassed` counts
|
|
@@ -93,15 +120,34 @@
|
|
|
93
120
|
* clears. The operator tells them apart, not the code.
|
|
94
121
|
* 4. Safe to kill and re-create: no disk state, no side effects.
|
|
95
122
|
* The cost of a crash is the dedup cache (see dedup.js).
|
|
123
|
+
* 5. No payload size cap and no intake rate cap. A flood of large
|
|
124
|
+
* messages is bounded only by the broker's own per-client limits
|
|
125
|
+
* and by TCP back-pressure on the socket. ADR-026 (proposed) is
|
|
126
|
+
* the home for a cap; the handbook states the limit.
|
|
127
|
+
*
|
|
128
|
+
* Performance, measured on the private benchmark harness (2026-04-24,
|
|
129
|
+
* M4 Max; pre-ADR-022 dedup, whose replacement has the same O(1)
|
|
130
|
+
* per-message profile): composer's own decode/dedup/dispatch path
|
|
131
|
+
* sustains 897 k msg/s at 100 B payloads (317 k at 1 KB). End-to-end
|
|
132
|
+
* through Mosquitto at QoS 1 the single-process ceiling was
|
|
133
|
+
* ~12.9 k msg/s, owned by mqtt.js and the broker round-trip, not this
|
|
134
|
+
* code. At 10 k msg/s steady state every cell ran clean; a
|
|
135
|
+
* 30 M-message run showed no subscriber-side leak.
|
|
96
136
|
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
137
|
+
* Decisions this file follows:
|
|
138
|
+
* - ADR-004 and ADR-013 — the message handler is synchronous and
|
|
139
|
+
* allocates nothing on the success path. The flow's dispatch never
|
|
140
|
+
* waits on it.
|
|
141
|
+
* - ADR-018 — the adapter contract: phases, status shape, codes.
|
|
142
|
+
* - ADR-022 — the dedup cache and its two bounds.
|
|
143
|
+
* - ADR-027 — the callback wrapper scope. `transform` runs under the
|
|
144
|
+
* shared guard. `onMessage` is not wrapped here; the flow's dispatch
|
|
145
|
+
* guard owns it.
|
|
146
|
+
* - ADR-028 — the grammar of every facade line printed here and in
|
|
147
|
+
* status.js.
|
|
148
|
+
* - ADR-030 — `localhost` is refused; a name is warned about.
|
|
149
|
+
* - ADR-026 (proposed) — the external-threat surface. TLS,
|
|
150
|
+
* authentication, and payload caps belong there, not here.
|
|
105
151
|
*
|
|
106
152
|
* @see src/core/source-manager/mqtt/status.js - The reporting rules
|
|
107
153
|
* @see src/core/emitter-manager/mqtt/emitter.js - Emitter counterpart
|
|
@@ -119,6 +165,50 @@ import {
|
|
|
119
165
|
} from './constants.js';
|
|
120
166
|
import { createDedupCache } from './dedup.js';
|
|
121
167
|
import { createStatusReporter } from './status.js';
|
|
168
|
+
import { isUsableRecord, describeShape } from '../record-shape.js';
|
|
169
|
+
import { wrapTransform, TRANSFORM_THREW } from '../../utils/callback-guard/index.js';
|
|
170
|
+
import { jitteredPeriod } from '../../utils/jitter/index.js';
|
|
171
|
+
import { logger } from '../../logger/index.js';
|
|
172
|
+
import { classifyAddress, localhostRefusalMessage, nameWarningMessage } from '../../utils/address/index.js';
|
|
173
|
+
|
|
174
|
+
// ============================================================================
|
|
175
|
+
// ADDRESS POLICY (ADR-030)
|
|
176
|
+
// ============================================================================
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Classifies the broker address and refuses `localhost`. The schema
|
|
180
|
+
* already refused it at flow definition; this call covers direct
|
|
181
|
+
* callers and carries the classified code. The source has no
|
|
182
|
+
* environment fallback for `brokerUrl`, so no env var is named.
|
|
183
|
+
*
|
|
184
|
+
* @param {string} brokerUrl - The broker URL as configured
|
|
185
|
+
* @returns {Object} The classified address
|
|
186
|
+
* @throws {Error} INVALID_CONFIG when the host is `localhost`
|
|
187
|
+
*/
|
|
188
|
+
const assertBrokerNotLocalhost = function ( brokerUrl ) {
|
|
189
|
+
const address = classifyAddress( brokerUrl, 'url' );
|
|
190
|
+
if ( address.kind === 'localhost' ) {
|
|
191
|
+
const err = new Error(
|
|
192
|
+
`winkComposer/mqttSource: ${localhostRefusalMessage( { field: 'brokerUrl', address } )}`
|
|
193
|
+
);
|
|
194
|
+
err.code = 'INVALID_CONFIG';
|
|
195
|
+
throw err;
|
|
196
|
+
}
|
|
197
|
+
return address;
|
|
198
|
+
}; // assertBrokerNotLocalhost()
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Prints the one ADDRESS_IS_NAME line when the broker host is a name.
|
|
202
|
+
*
|
|
203
|
+
* @param {Object} address - The classified broker address
|
|
204
|
+
*/
|
|
205
|
+
const warnIfBrokerIsName = function ( address ) {
|
|
206
|
+
if ( address.kind === 'name' ) {
|
|
207
|
+
logger.warn(
|
|
208
|
+
`winkComposer/mqttSource: ${nameWarningMessage( { field: 'brokerUrl', host: address.host } )}`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
}; // warnIfBrokerIsName()
|
|
122
212
|
|
|
123
213
|
// ============================================================================
|
|
124
214
|
// CLIENT FACTORY
|
|
@@ -129,11 +219,14 @@ import { createStatusReporter } from './status.js';
|
|
|
129
219
|
* health/metrics reporting.
|
|
130
220
|
*
|
|
131
221
|
* @param {Object} config - Client configuration
|
|
132
|
-
* @param {string} config.brokerUrl - MQTT broker URL (e.g., 'mqtt://
|
|
222
|
+
* @param {string} config.brokerUrl - MQTT broker URL (e.g., 'mqtt://127.0.0.1:1883'; never localhost)
|
|
133
223
|
* @param {string|string[]} config.topics - Topic(s) to subscribe to (supports wildcards)
|
|
134
224
|
* @param {function} config.onMessage - Message handler: (message) => void
|
|
135
225
|
* @param {Object} [config.codec] - Codec for payload decoding (default: JSON.parse)
|
|
136
|
-
* @param {function} [config.transform] - Optional message transform:
|
|
226
|
+
* @param {function} [config.transform] - Optional message transform:
|
|
227
|
+
* (msg) => transformedMsg; return null/undefined to drop (counted in
|
|
228
|
+
* skipped); a throw or a scalar/array return skips the message
|
|
229
|
+
* (CALLBACK_FAILED) and the stream continues
|
|
137
230
|
* @param {number} [config.dedupWindowMs=120000] - Dedup time bound (ADR-022)
|
|
138
231
|
* @param {number} [config.dedupMaxEntries=65536] - Dedup count cap (ADR-022)
|
|
139
232
|
* @param {string} [config.clientId] - MQTT client ID (auto-generated if omitted)
|
|
@@ -168,17 +261,20 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
168
261
|
// Validate required config. Per ADR-018, setup-time throws carry
|
|
169
262
|
// classified err.code (INVALID_CONFIG for missing/malformed config fields).
|
|
170
263
|
if ( !brokerUrl ) {
|
|
171
|
-
const err = new Error( '
|
|
264
|
+
const err = new Error( 'winkComposer/mqttSource: brokerUrl is required' );
|
|
172
265
|
err.code = 'INVALID_CONFIG';
|
|
173
266
|
throw err;
|
|
174
267
|
}
|
|
268
|
+
// Address policy (ADR-030): `localhost` is refused before the
|
|
269
|
+
// client is created; a name is warned about just before connect.
|
|
270
|
+
const brokerAddress = assertBrokerNotLocalhost( brokerUrl );
|
|
175
271
|
if ( !topics || ( Array.isArray( topics ) && topics.length === 0 ) ) {
|
|
176
|
-
const err = new Error( '
|
|
272
|
+
const err = new Error( 'winkComposer/mqttSource: topics is required' );
|
|
177
273
|
err.code = 'INVALID_CONFIG';
|
|
178
274
|
throw err;
|
|
179
275
|
}
|
|
180
276
|
if ( typeof onMessage !== 'function' ) {
|
|
181
|
-
const err = new Error( '
|
|
277
|
+
const err = new Error( 'winkComposer/mqttSource: onMessage must be a function' );
|
|
182
278
|
err.code = 'INVALID_CONFIG';
|
|
183
279
|
throw err;
|
|
184
280
|
}
|
|
@@ -201,18 +297,34 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
201
297
|
|
|
202
298
|
reporter.starting();
|
|
203
299
|
|
|
204
|
-
//
|
|
205
|
-
|
|
300
|
+
// The user's transform runs under the shared callback guard, armed
|
|
301
|
+
// once here — never per message. A throw inside it becomes the
|
|
302
|
+
// sentinel plus one per-record CALLBACK_FAILED report through the
|
|
303
|
+
// reporter; the message handler turns the sentinel into a skip.
|
|
304
|
+
// The topic travels as the guard's per-call context, so the
|
|
305
|
+
// success path allocates nothing (ADR-018).
|
|
306
|
+
const reportTransformFault = function ( detail, topic ) {
|
|
307
|
+
reporter.transformFailed( `topic '${topic}': transform threw: ${detail} — message skipped` );
|
|
308
|
+
};
|
|
309
|
+
const guardedTransform = transform ? wrapTransform( transform, reportTransformFault ) : null;
|
|
310
|
+
|
|
311
|
+
// The auto-generated name carries the start time and a random
|
|
312
|
+
// part, the emitter's shape. Two sources that start in the same
|
|
313
|
+
// millisecond still get different names. That matters because the
|
|
314
|
+
// broker allows one live connection per name. When a second client
|
|
315
|
+
// arrives under a name in use, the broker disconnects the first.
|
|
316
|
+
// Two clients then take the session from each other on every
|
|
317
|
+
// reconnect.
|
|
318
|
+
const generatedClientId = clientId || `wink-source-${Date.now()}-${Math.random().toString( 36 ).slice( 2, 9 )}`;
|
|
206
319
|
|
|
207
320
|
// A persistent session is filed at the broker under the client's
|
|
208
321
|
// name. The auto-generated name changes on every start, so the
|
|
209
|
-
// backlog
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
// only — nothing here touches the message path.
|
|
322
|
+
// backlog saved under the previous run's name is never delivered.
|
|
323
|
+
// Warn once at startup. The fix is one config line, on the setup
|
|
324
|
+
// path only.
|
|
213
325
|
if ( !clientId && cleanStart !== true ) {
|
|
214
|
-
|
|
215
|
-
'
|
|
326
|
+
logger.warn(
|
|
327
|
+
'winkComposer/mqttSource: no clientId configured — this session ' +
|
|
216
328
|
`is persistent under the auto-generated name '${generatedClientId}'. ` +
|
|
217
329
|
'After a restart, composer connects under a NEW name, so messages ' +
|
|
218
330
|
'the broker saved during the downtime are never delivered. Set a ' +
|
|
@@ -220,10 +332,14 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
220
332
|
);
|
|
221
333
|
}
|
|
222
334
|
|
|
223
|
-
// Build MQTT options, allowing cleanStart override
|
|
335
|
+
// Build MQTT options, allowing cleanStart override. The reconnect
|
|
336
|
+
// period is the configured one plus a random share of up to 20%,
|
|
337
|
+
// drawn once here. So a fleet that lost one broker does not retry
|
|
338
|
+
// in step. The configured period is the floor.
|
|
224
339
|
const mqttOptions = {
|
|
225
340
|
...MQTT_SOURCE_CONFIG,
|
|
226
|
-
clientId: generatedClientId
|
|
341
|
+
clientId: generatedClientId,
|
|
342
|
+
reconnectPeriod: jitteredPeriod( MQTT_SOURCE_CONFIG.reconnectPeriod )
|
|
227
343
|
};
|
|
228
344
|
|
|
229
345
|
// Map the user-facing `cleanStart` key (MQTT 5 term) onto the
|
|
@@ -233,7 +349,9 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
233
349
|
mqttOptions.clean = cleanStart;
|
|
234
350
|
}
|
|
235
351
|
|
|
236
|
-
// Create MQTT client
|
|
352
|
+
// Create MQTT client. The name warning goes first, so it prints
|
|
353
|
+
// before any socket opens (ADR-030).
|
|
354
|
+
warnIfBrokerIsName( brokerAddress );
|
|
237
355
|
const client = mqttConnectFn( brokerUrl, mqttOptions );
|
|
238
356
|
|
|
239
357
|
// One heartbeat per second: re-evaluates the time-based health
|
|
@@ -272,7 +390,10 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
272
390
|
const dedupId = userProps ? userProps[ WINK_NAMESPACE.dedupId ] : undefined;
|
|
273
391
|
|
|
274
392
|
// Dedup is opt-in by construction (ADR-022): no id → bypass.
|
|
275
|
-
|
|
393
|
+
// A repeated MQTT 5 user property parses to an array, and an
|
|
394
|
+
// array can never equal an earlier one. So any non-string id
|
|
395
|
+
// bypasses too, counted in dedupBypassed rather than cached.
|
|
396
|
+
if ( typeof dedupId !== 'string' ) {
|
|
276
397
|
reporter.bypassed();
|
|
277
398
|
} else if ( dedup.isDuplicate( dedupId ) ) {
|
|
278
399
|
reporter.dupSkipped();
|
|
@@ -281,9 +402,17 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
281
402
|
reporter.idAccepted();
|
|
282
403
|
}
|
|
283
404
|
|
|
284
|
-
// Decode payload
|
|
285
|
-
//
|
|
405
|
+
// Decode the payload, check its shape, and attach metadata in
|
|
406
|
+
// one guarded region. A failure anywhere in it is skipped,
|
|
407
|
+
// classified, and reported per record (ADR-018). The stream
|
|
286
408
|
// continues.
|
|
409
|
+
//
|
|
410
|
+
// The shape guard is needed because a valid JSON document can
|
|
411
|
+
// be a scalar or a bare array. The attach is inside the guard
|
|
412
|
+
// because a codec can return a frozen record, or one with a
|
|
413
|
+
// non-writable _topic. The assignment then throws in strict
|
|
414
|
+
// mode. decodeOk() runs only when the whole record survived,
|
|
415
|
+
// so the ring gets exactly one entry per message.
|
|
287
416
|
let message;
|
|
288
417
|
try {
|
|
289
418
|
if ( codec && typeof codec.unpack === 'function' ) {
|
|
@@ -291,35 +420,65 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
291
420
|
} else {
|
|
292
421
|
message = JSON.parse( payload.toString() );
|
|
293
422
|
}
|
|
423
|
+
|
|
424
|
+
if ( !isUsableRecord( message ) ) {
|
|
425
|
+
reporter.decodeFailed( `topic '${topic}': payload decoded to ${describeShape( message )} — a record object is required — message skipped` );
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// Attach metadata for downstream use
|
|
430
|
+
message._topic = topic; // eslint-disable-line no-underscore-dangle
|
|
431
|
+
message._dedupId = dedupId; // eslint-disable-line no-underscore-dangle
|
|
432
|
+
|
|
294
433
|
reporter.decodeOk();
|
|
295
434
|
} catch ( err ) {
|
|
296
|
-
|
|
435
|
+
// The parser's own message can echo a fragment of the
|
|
436
|
+
// payload. Payload text is data, so it stays below the
|
|
437
|
+
// warn level (ADR-028). The report names the topic and
|
|
438
|
+
// the size. The parser's reason goes to a debug line,
|
|
439
|
+
// guarded so the hot path builds nothing when debug is off.
|
|
440
|
+
reporter.decodeFailed( `topic '${topic}': payload of ${payload.length} bytes could not be decoded — message skipped` );
|
|
441
|
+
if ( logger.debugOn ) {
|
|
442
|
+
logger.debug( `winkComposer/mqttSource: decode failed [DECODE_ERROR]: topic '${topic}': ${err.message}` );
|
|
443
|
+
}
|
|
297
444
|
return;
|
|
298
445
|
}
|
|
299
446
|
|
|
300
|
-
//
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
//
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
//
|
|
447
|
+
// Apply the optional transform, pre-armed by the shared guard
|
|
448
|
+
// at startup. A throw skips this one message with a per-record
|
|
449
|
+
// CALLBACK_FAILED report, and the stream continues. User code
|
|
450
|
+
// must never propagate into mqtt.js's event processing
|
|
451
|
+
// (transform contract, 2026-07-11).
|
|
452
|
+
//
|
|
453
|
+
// The return is held to the same record shape as the payload.
|
|
454
|
+
// A null or undefined return stays the intentional silent
|
|
455
|
+
// drop. A scalar or array return is one per-record
|
|
456
|
+
// CALLBACK_FAILED. The shape check runs only when a transform
|
|
457
|
+
// is configured. The plain path pays nothing.
|
|
308
458
|
let finalMessage;
|
|
309
|
-
|
|
310
|
-
finalMessage =
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
459
|
+
if ( guardedTransform ) {
|
|
460
|
+
finalMessage = guardedTransform( message, topic );
|
|
461
|
+
// The sentinel check runs FIRST: the sentinel is a plain
|
|
462
|
+
// object, so a later isUsableRecord check would wave it
|
|
463
|
+
// through into onMessage as a message.
|
|
464
|
+
if ( finalMessage === TRANSFORM_THREW ) {
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
if ( finalMessage === null || finalMessage === undefined ) {
|
|
468
|
+
reporter.transformDropped();
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
if ( !isUsableRecord( finalMessage ) ) {
|
|
472
|
+
reporter.transformFailed( `topic '${topic}': transform returned ${describeShape( finalMessage )} — a record object (or null/undefined to drop) is required — message skipped` );
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
} else {
|
|
476
|
+
finalMessage = message;
|
|
314
477
|
}
|
|
315
478
|
|
|
316
479
|
// Deliver to handler
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
reporter.delivered();
|
|
320
|
-
} else {
|
|
321
|
-
reporter.transformDropped();
|
|
322
|
-
}
|
|
480
|
+
onMessage( finalMessage );
|
|
481
|
+
reporter.delivered();
|
|
323
482
|
} );
|
|
324
483
|
|
|
325
484
|
client.on( 'offline', function () {
|
|
@@ -339,51 +498,85 @@ const createMQTTSourceClient = function ( config ) {
|
|
|
339
498
|
// STOP FUNCTION
|
|
340
499
|
// ========================================================================
|
|
341
500
|
|
|
501
|
+
// The stop outcome is latched on the first call. A second caller
|
|
502
|
+
// receives the same promise, so it can never report a clean stop
|
|
503
|
+
// while the first call is still closing.
|
|
504
|
+
let stopPromise = null;
|
|
505
|
+
|
|
342
506
|
/**
|
|
343
507
|
* Stop the MQTT client, with a time budget.
|
|
344
508
|
*
|
|
345
|
-
* Per ADR-018, source stop functions take a `{ timeout }`.
|
|
346
|
-
*
|
|
347
|
-
* which waits for any messages still being sent. If that does not
|
|
348
|
-
* finish within `timeout` ms, we ask for a forced disconnect
|
|
349
|
-
* (`client.end(true)`), which closes the socket right away — so the
|
|
350
|
-
* rest of the shutdown can proceed even if the broker is slow or
|
|
351
|
-
* unreachable. The forced path is reported with the sources' shared
|
|
352
|
-
* `note` convention.
|
|
509
|
+
* Per ADR-018, source stop functions take a `{ timeout }`. How the
|
|
510
|
+
* close detaches follows the library's own rules (mqtt.js 5.15.1):
|
|
353
511
|
*
|
|
354
|
-
*
|
|
355
|
-
*
|
|
512
|
+
* - Not connected (the first connect still pending, the link down,
|
|
513
|
+
* or the library waiting to retry): `end( true )` at once. The
|
|
514
|
+
* library destroys the stream, clears its connect and reconnect
|
|
515
|
+
* timers, and calls back on the next tick. A graceful
|
|
516
|
+
* `end( false )` would call back at once here too, but it would
|
|
517
|
+
* not destroy the stream (`client.js:921-924`). A pending
|
|
518
|
+
* connect would then hold the process for the whole connect
|
|
519
|
+
* timeout.
|
|
520
|
+
* - Connected: `end( false )` sends DISCONNECT and half-closes the
|
|
521
|
+
* socket, then waits for the broker to close its side. A timer
|
|
522
|
+
* bounds that wait. At the budget the timer destroys the stream
|
|
523
|
+
* directly. That is the call the library's own connect timeout
|
|
524
|
+
* makes. A second `end()` would do nothing: it returns at once
|
|
525
|
+
* behind the library's `disconnecting` flag (`client.js:731-734`).
|
|
526
|
+
* The forced path is reported with the sources' shared `note`
|
|
527
|
+
* convention.
|
|
356
528
|
*
|
|
357
|
-
*
|
|
529
|
+
* The timer is `unref()`ed so it does not keep Node alive while a
|
|
530
|
+
* clean disconnect is in progress. Either path settles the promise
|
|
531
|
+
* once. The default time budget (5000 ms) matches the sinks.
|
|
358
532
|
*
|
|
359
533
|
* @param {Object} [options] - Stop options
|
|
360
|
-
* @param {number} [options.timeout=5000] - Max ms to wait for clean disconnect
|
|
534
|
+
* @param {number} [options.timeout=5000] - Max ms to wait for the clean disconnect
|
|
361
535
|
* @returns {Promise<void>} Resolves once the client is closed (clean or forced)
|
|
362
536
|
*/
|
|
363
537
|
const stop = function ( { timeout = 5000 } = {} ) {
|
|
538
|
+
if ( stopPromise ) {
|
|
539
|
+
return stopPromise;
|
|
540
|
+
}
|
|
364
541
|
clearInterval( cadence );
|
|
365
|
-
|
|
542
|
+
stopPromise = new Promise( function ( resolve ) {
|
|
366
543
|
let settled = false;
|
|
367
|
-
|
|
368
|
-
const onClosed = function () {
|
|
369
|
-
if ( settled ) return;
|
|
370
|
-
settled = true;
|
|
371
|
-
if ( forceTimer ) clearTimeout( forceTimer );
|
|
544
|
+
const reportClean = function () {
|
|
372
545
|
reporter.stopped();
|
|
546
|
+
};
|
|
547
|
+
const reportForced = function () {
|
|
548
|
+
reporter.stopForced( timeout );
|
|
549
|
+
};
|
|
550
|
+
// The library's close callback also fires after the timer's
|
|
551
|
+
// destroy, so the second arrival must settle nothing.
|
|
552
|
+
const settle = function ( report ) {
|
|
553
|
+
if ( settled ) {
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
settled = true;
|
|
557
|
+
report();
|
|
373
558
|
resolve();
|
|
374
559
|
};
|
|
375
|
-
forceTimer = setTimeout( function () {
|
|
376
560
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
561
|
+
if ( !client.connected ) {
|
|
562
|
+
client.end( true, {}, function () {
|
|
563
|
+
settle( reportClean );
|
|
564
|
+
} );
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
const forceTimer = setTimeout( function () {
|
|
569
|
+
client.stream.destroy();
|
|
570
|
+
settle( reportForced );
|
|
381
571
|
}, timeout );
|
|
382
|
-
// Don't keep Node alive on the timer while the broker is
|
|
383
|
-
// closing cleanly. Either path resolves the Promise.
|
|
384
572
|
forceTimer.unref();
|
|
385
|
-
|
|
573
|
+
|
|
574
|
+
client.end( false, {}, function () {
|
|
575
|
+
clearTimeout( forceTimer );
|
|
576
|
+
settle( reportClean );
|
|
577
|
+
} );
|
|
386
578
|
} );
|
|
579
|
+
return stopPromise;
|
|
387
580
|
};
|
|
388
581
|
|
|
389
582
|
// Expose for testing
|
|
@@ -28,8 +28,8 @@
|
|
|
28
28
|
* - Two unavoidable residuals, both bounded: (1) `Set.add`/`Set.delete`
|
|
29
29
|
* maintain V8's internal hash table — amortized O(1), memory bounded
|
|
30
30
|
* by `maxEntries`, and the identical profile the pre-ADR-022 cache
|
|
31
|
-
* already had (JS offers no pre-sized Set). (2)
|
|
32
|
-
*
|
|
31
|
+
* already had (JS offers no pre-sized Set). (2) The clock returns a
|
|
32
|
+
* fractional number, so the timestamp is transiently boxed; it
|
|
33
33
|
* unboxes on store into the Float64Array and dies in the nursery.
|
|
34
34
|
* - Eviction clears the ring slot so an evicted id string is not
|
|
35
35
|
* retained past its window — that release is deliberate.
|
|
@@ -48,9 +48,12 @@
|
|
|
48
48
|
* -----------
|
|
49
49
|
* 1. A dedup id is unique per message and repeats only on a QoS-1
|
|
50
50
|
* retransmission (the emitter stamps UUIDs).
|
|
51
|
-
* 2. The clock
|
|
52
|
-
* clock
|
|
53
|
-
*
|
|
51
|
+
* 2. The clock counts up, in milliseconds. The default is the
|
|
52
|
+
* stopwatch clock (`performance`), which a wall-clock step cannot
|
|
53
|
+
* move. A forward step of the wall clock would otherwise expire
|
|
54
|
+
* every entry at once; a backward step would hold entries one
|
|
55
|
+
* step too long (ADR-018, long-running stability). An injected
|
|
56
|
+
* `nowFn` must count up too.
|
|
54
57
|
*
|
|
55
58
|
* LIMITATIONS
|
|
56
59
|
* -----------
|
|
@@ -62,9 +65,9 @@
|
|
|
62
65
|
* effective window is maxEntries ÷ rate (the full 120 s up to
|
|
63
66
|
* about 550 msg/s at defaults; ~6.5 s at 10 k msg/s). Stated in
|
|
64
67
|
* ADR-022, not hidden.
|
|
65
|
-
* 3. Memory, measured (
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
+
* 3. Memory, measured on the private benchmark harness (2026-07-09):
|
|
69
|
+
* 86 bytes per production-shaped id, ~6.7 MB worst case at the
|
|
70
|
+
* 65,536 cap including bounded V8 Set slack.
|
|
68
71
|
*
|
|
69
72
|
* @see ADR-022 - The full decision
|
|
70
73
|
*/
|
|
@@ -73,6 +76,7 @@ import {
|
|
|
73
76
|
DEFAULT_DEDUP_WINDOW_MS,
|
|
74
77
|
DEFAULT_DEDUP_MAX_ENTRIES
|
|
75
78
|
} from './constants.js';
|
|
79
|
+
import { monotonicNow } from '../../utils/clock/index.js';
|
|
76
80
|
|
|
77
81
|
// ============================================================================
|
|
78
82
|
// VALIDATION HELPER
|
|
@@ -87,7 +91,7 @@ import {
|
|
|
87
91
|
*/
|
|
88
92
|
const assertPositiveInteger = function ( value, name ) {
|
|
89
93
|
if ( typeof value !== 'number' || value < 1 || !Number.isInteger( value ) ) {
|
|
90
|
-
const err = new Error( `
|
|
94
|
+
const err = new Error( `winkComposer/mqttSource: ${name} must be a positive integer` );
|
|
91
95
|
err.code = 'INVALID_CONFIG';
|
|
92
96
|
throw err;
|
|
93
97
|
}
|
|
@@ -108,13 +112,14 @@ const assertPositiveInteger = function ( value, name ) {
|
|
|
108
112
|
* expires once its age reaches this many milliseconds
|
|
109
113
|
* @param {number} [options.maxEntries=65536] - Count cap: the memory
|
|
110
114
|
* guarantee; the oldest entry is evicted when the ring is full
|
|
111
|
-
* @param {function} [options.nowFn=
|
|
112
|
-
*
|
|
115
|
+
* @param {function} [options.nowFn=monotonicNow] - Clock source, in
|
|
116
|
+
* milliseconds and counting up. Injection point for deterministic
|
|
117
|
+
* tests; production uses the stopwatch clock
|
|
113
118
|
* @returns {Object} Cache instance with isDuplicate, size, clear, has
|
|
114
119
|
*/
|
|
115
120
|
const createDedupCache = function ( options = {} ) {
|
|
116
121
|
if ( typeof options !== 'object' || options === null ) {
|
|
117
|
-
const err = new Error( '
|
|
122
|
+
const err = new Error( 'winkComposer/mqttSource: dedup options must be an object' );
|
|
118
123
|
err.code = 'INVALID_CONFIG';
|
|
119
124
|
throw err;
|
|
120
125
|
}
|
|
@@ -122,13 +127,13 @@ const createDedupCache = function ( options = {} ) {
|
|
|
122
127
|
const {
|
|
123
128
|
windowMs = DEFAULT_DEDUP_WINDOW_MS,
|
|
124
129
|
maxEntries = DEFAULT_DEDUP_MAX_ENTRIES,
|
|
125
|
-
nowFn =
|
|
130
|
+
nowFn = monotonicNow
|
|
126
131
|
} = options;
|
|
127
132
|
|
|
128
133
|
assertPositiveInteger( windowMs, 'windowMs' );
|
|
129
134
|
assertPositiveInteger( maxEntries, 'maxEntries' );
|
|
130
135
|
if ( typeof nowFn !== 'function' ) {
|
|
131
|
-
const err = new Error( '
|
|
136
|
+
const err = new Error( 'winkComposer/mqttSource: nowFn must be a function' );
|
|
132
137
|
err.code = 'INVALID_CONFIG';
|
|
133
138
|
throw err;
|
|
134
139
|
}
|