@winkjs/composer 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/README.md +1 -1
  2. package/package.json +3 -3
  3. package/src/composer.js +1 -1
  4. package/src/core/batch/aoo-encoder.js +4 -4
  5. package/src/core/batch/sliding-window.js +2 -2
  6. package/src/core/codec/index.js +4 -4
  7. package/src/core/emitter-manager/mqtt/connection.js +209 -0
  8. package/src/core/emitter-manager/mqtt/constants.js +43 -29
  9. package/src/core/emitter-manager/mqtt/emitter.js +108 -633
  10. package/src/core/emitter-manager/mqtt/health.js +151 -0
  11. package/src/core/emitter-manager/mqtt/index.js +68 -4
  12. package/src/core/emitter-manager/mqtt/publish.js +292 -0
  13. package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
  14. package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
  15. package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
  16. package/src/core/emitter-manager/terminal/index.js +20 -5
  17. package/src/core/env-vars.js +143 -16
  18. package/src/core/logger/index.js +174 -0
  19. package/src/core/mqtt-protocol.js +2 -2
  20. package/src/core/partition-manager/get-stats.js +31 -0
  21. package/src/core/partition-manager/index.js +1 -0
  22. package/src/core/partition-manager/init.js +15 -0
  23. package/src/core/partition-manager/update.js +143 -87
  24. package/src/core/partition-manager/validate-flow.js +1 -1
  25. package/src/core/semantics/loader.js +1 -1
  26. package/src/core/semantics/warnings.js +5 -3
  27. package/src/core/shutdown-manager/index.js +71 -21
  28. package/src/core/source-manager/csv/start.js +109 -35
  29. package/src/core/source-manager/mqtt/client.js +276 -83
  30. package/src/core/source-manager/mqtt/dedup.js +19 -14
  31. package/src/core/source-manager/mqtt/index.js +33 -6
  32. package/src/core/source-manager/mqtt/status.js +211 -46
  33. package/src/core/source-manager/record-shape.js +55 -0
  34. package/src/core/source-manager/test-harness/field-generator.js +1 -1
  35. package/src/core/source-manager/test-harness/start.js +52 -10
  36. package/src/core/source-manager/test-harness/validate.js +1 -1
  37. package/src/core/storage-manager/questdb/address-policy.js +264 -0
  38. package/src/core/storage-manager/questdb/assert-columns.js +2 -2
  39. package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
  40. package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
  41. package/src/core/storage-manager/questdb/flush-engine.js +852 -0
  42. package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
  43. package/src/core/storage-manager/questdb/index.js +592 -827
  44. package/src/core/storage-manager/questdb/persist-plan.js +147 -162
  45. package/src/core/storage-manager/questdb/resolve-options.js +403 -0
  46. package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
  47. package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
  48. package/src/core/test-utils/tcp-proxy.js +194 -1
  49. package/src/core/tunable/helpers.js +1 -1
  50. package/src/core/utils/address/index.js +362 -0
  51. package/src/core/utils/address/probe.js +210 -0
  52. package/src/core/utils/callback-guard/index.js +176 -0
  53. package/src/core/utils/clock/index.js +34 -0
  54. package/src/core/utils/half-life/index.js +4 -4
  55. package/src/core/utils/jitter/index.js +51 -0
  56. package/src/core/utils/line-rate/index.js +104 -0
  57. package/src/core/utils/quantize/index.js +43 -19
  58. package/src/core/utils/validate/composer-validators.js +2 -2
  59. package/src/core/utils/validate/index.js +1 -1
  60. package/src/core/utils/validate/validate.js +1 -1
  61. package/src/core/wiring/assert-handle.js +5 -5
  62. package/src/core/wiring/assert-module.js +2 -2
  63. package/src/core/wiring/wire-emitters.js +7 -4
  64. package/src/core/wiring/wire-linear-graph.js +1 -1
  65. package/src/core/wiring/wire-node.js +6 -6
  66. package/src/core/wiring/wire-semantics.js +1 -1
  67. package/src/core/wiring/wire-storages.js +14 -5
  68. package/src/flow/driver.js +31 -6
  69. package/src/flow/expand-single-input-array.js +4 -4
  70. package/src/flow/flow-api-schemas.js +4 -4
  71. package/src/flow/flow.js +62 -41
  72. package/src/flow/for-each.js +12 -11
  73. package/src/flow/get-target-array.js +2 -2
  74. package/src/flow/load-node-module.js +1 -1
  75. package/src/flow/make-node-method.js +4 -4
  76. package/src/flow/resolve-pick-by-field.js +1 -1
  77. package/src/flow/run.js +200 -40
  78. package/src/flow/serialize.js +1 -1
  79. package/src/flow/spec-builder.js +1 -1
  80. package/src/flow/validate.js +1 -1
  81. package/src/nodes/butterworth-filter/init.js +5 -4
  82. package/src/nodes/categorize/update.js +3 -1
  83. package/src/nodes/controller/update.js +2 -1
  84. package/src/nodes/dwell-time-tracker/update.js +2 -1
  85. package/src/nodes/emit-if/helpers.js +57 -11
  86. package/src/nodes/emit-if/init.js +1 -1
  87. package/src/nodes/emit-if/update.js +8 -11
  88. package/src/nodes/page-hinkley/update.js +2 -1
  89. package/src/nodes/pass-if/update.js +3 -1
  90. package/src/nodes/persist-if/helpers.js +60 -4
  91. package/src/nodes/persist-if/init.js +1 -1
  92. package/src/nodes/persist-if/update.js +8 -16
  93. package/src/nodes/persistence-check/update.js +2 -1
  94. package/src/nodes/sanitize/update.js +4 -2
  95. package/src/nodes/swing-watch/update.js +3 -2
  96. package/src/nodes/threshold/update.js +2 -1
  97. package/src/nodes/trend/compute-confidence.js +4 -4
  98. package/src/nodes/trend/update.js +2 -1
  99. package/src/nodes/winnow/update.js +3 -1
  100. package/src/tools/training/features.js +13 -13
  101. package/src/tools/training/logistic.js +1 -1
  102. package/src/tools/training/metrics.js +2 -2
  103. package/src/tools/training/scale.js +2 -2
  104. package/src/tools/training/validate.js +7 -7
  105. package/src/core/utils/console-formatter/index.js +0 -207
  106. package/src/nodes/archive/csv/index.js +0 -6
  107. package/src/nodes/archive/csv/init.js +0 -36
  108. package/src/nodes/archive/csv/load.js +0 -70
  109. package/src/nodes/archive/csv/reset.js +0 -14
  110. package/src/nodes/archive/csv/simple-parser.js +0 -216
  111. package/src/nodes/archive/csv/test.js +0 -173
  112. package/src/nodes/archive/csv/update.js +0 -63
  113. package/src/nodes/sse-emitter/index.js +0 -7
  114. package/src/nodes/sse-emitter/init.js +0 -105
  115. package/src/nodes/sse-emitter/introspect.js +0 -134
  116. package/src/nodes/sse-emitter/publish-to.js +0 -6
  117. package/src/nodes/sse-emitter/recompute.js +0 -16
  118. package/src/nodes/sse-emitter/reset.js +0 -19
  119. package/src/nodes/sse-emitter/update.js +0 -69
  120. package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
  121. package/src/nodes/sse-emitter/utils/build-message.js +0 -31
  122. package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
  123. package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
  124. package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
  125. package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
@@ -15,6 +15,20 @@
15
15
  * closes that gap by construction; the composer-owned WAL that restores
16
16
  * crash durability is ADR-021's planned successor.
17
17
  *
18
+ * The adapter is one closure factory per concern, the shape QuestDB's
19
+ * flush engine uses. This file is the factory. It resolves the
20
+ * configuration, builds the state, opens the client, and assembles the
21
+ * handle from the modules beside it:
22
+ * - `resolve-config.js` validates every option in one fixed order and
23
+ * arms the three callbacks through the shared guard.
24
+ * - `connection.js` opens the client, waits for the first connack, and
25
+ * keeps `state.connected` and the counters current.
26
+ * - `health.js` reads pressure and health, and signals backpressure.
27
+ * - `publish.js` is the hot path, `publishNow`.
28
+ * - `shutdown-drain.js` is the latched drain-then-close shutdown.
29
+ * Each module captures what it needs at setup. Nothing moves onto a
30
+ * per-message argument list.
31
+ *
18
32
  * DESIGN DECISIONS:
19
33
  *
20
34
  * 1. FIRE-AND-FORGET WITH QoS 1
@@ -37,6 +51,12 @@
37
51
  * 3. MESSAGE EXPIRY (MQTT v5)
38
52
  * - Configurable per message type
39
53
  * - Prevents flooding backend after extended offline periods
54
+ * - The type lookup is a null-prototype table. A prototype key such
55
+ * as 'constructor' would hand the library a function, and
56
+ * mqtt-packet then destroys the stream with an error that carries
57
+ * no code. mqtt.js swallows such an error, so no handler sees it,
58
+ * and the poison packet replays on every reconnect. An unknown
59
+ * key falls back to the default expiry instead.
40
60
  *
41
61
  * 4. NO FORCED IDENTITY / OPTIONAL WILL — unchanged transport behavior.
42
62
  *
@@ -48,8 +68,10 @@
48
68
  * reachable broker the real wait is one connack round trip; on an
49
69
  * unreachable one the flow starts with `connected: false` and the
50
70
  * client keeps retrying in the background.
51
- * - Return shape: the handle itself at grace 0, otherwise a Promise
52
- * of it. The wiring layer awaits either (ADR-018).
71
+ * - Return shape: always a Promise of the handle, as QuestDB's
72
+ * factory returns. At grace 0 the promise resolves without a
73
+ * wait. A configuration refusal arrives as a rejection carrying
74
+ * the classified code. The wiring layer awaits it (ADR-018).
53
75
  * - This replaced the wire-time `sleep(240)` that once papered over
54
76
  * the pre-connack loss ADR-021 eliminated.
55
77
  *
@@ -61,7 +83,12 @@
61
83
  * when supplied (ctx = `{ topic }`); without a handler the adapter
62
84
  * surfaces the failure via `Promise.reject(deliveryErr)` — loud failure
63
85
  * beats silent loss (parity with QuestDB). `onCritical` is
64
- * reserved for the high-pressure `QUEUE_CRITICAL` warning (no loss yet).
86
+ * reserved for the high-pressure `QUEUE_CRITICAL` warning (no loss
87
+ * yet). It fires once per climb past 0.8 and re-arms below 0.66;
88
+ * `onBackpressure` reports the pressure after every acknowledgment.
89
+ * - `flush( { timeout }? )` waits for every accepted message to settle,
90
+ * closing nothing and refusing nothing. On its deadline it rejects
91
+ * `DELIVERY_FAILED` with `pending: { count }` (ADR-018 §6).
65
92
  * - `getPressure()` returns unacked / maxQueueSize in `[0, 1]` (sync,
66
93
  * O(1), allocation-free per ADR-018).
67
94
  * - `getHealth()` returns the ADR-018 health floor `{status, connected,
@@ -100,223 +127,67 @@
100
127
  * - `SHUTTING_DOWN` — emitter is mid-shutdown; new publishes are dropped.
101
128
  * - `INVALID_CONFIG` — setup-time; missing or malformed config field
102
129
  * (brokerUrl, codec.pack, callback type, connectGraceMs). On the
103
- * thrown TypeError per ADR-018's fail-fast setup rule.
130
+ * thrown TypeError per ADR-018's fail-fast setup rule. Includes a
131
+ * `brokerUrl` whose host is `localhost` (ADR-030): the name can
132
+ * resolve to two addresses and the broker may listen on only one.
133
+ * The message names the literal to set. Refused before the client
134
+ * is created, at the schema (flow definition) and here (the
135
+ * `MQTT_BROKER_URL` fallback and direct callers).
104
136
  * - `SHUTDOWN_TIMEOUT` — shutdown closed with unacknowledged messages;
105
137
  * carries `dropped: { count }` (the exact counter value). Fires whether
106
138
  * connected or not: with no disk store, nothing survives the process,
107
139
  * so a disconnected shutdown with pending messages is a real loss and
108
140
  * is reported as one (this differs from the wal-backed design, which
109
141
  * held them for the next session).
142
+ * - `CALLBACK_FAILED` — a user callback (`onDeliveryFailure`,
143
+ * `onCritical`, `onBackpressure`) itself threw or rejected. The
144
+ * shared callback guard contains the fault (ADR-018): the emitter
145
+ * keeps publishing and each fault becomes one classified console
146
+ * line. Fix the callback; the line names it and carries the detail.
147
+ *
148
+ * Console classification (a token on a log line, not an `err.code`):
149
+ * - `DELIVERY_HEALTH` — the link changed state. One line per change,
150
+ * with or without `debug`: going offline at error, the next connack
151
+ * at warn with the outage length. Both carry the in-flight count.
152
+ * Nothing prints while a state persists (see `connection.js`).
153
+ * - `CONNECT_FAILED` — the client's `error` event, at warn, with
154
+ * the reason (a refused connect, a connack timeout). Bounded: two
155
+ * per episode in full, then one summary a minute.
156
+ * - `ADDRESS_IS_NAME` — the `brokerUrl` host is a name other than
157
+ * `localhost`. One `logger.warn` line at setup, before the client is
158
+ * created (ADR-030). A name is allowed and the emitter proceeds; the
159
+ * line tells the operator that only a literal address is immune to a
160
+ * resolver that changes its answer under a running process. This
161
+ * emitter refuses and warns; it does not probe, because its posture
162
+ * is recovering (ADR-018 §5).
110
163
  *
111
164
  * @module mqtt-emitter
112
165
  */
113
166
 
114
- import mqtt from 'mqtt';
115
- import crypto from 'crypto';
116
- import { ENV_VARS } from '../../env-vars.js';
117
- import {
118
- QOS,
119
- MQTT_CONFIG,
120
- WINK_NAMESPACE,
121
- QUEUE_CRITICAL_THRESHOLD,
122
- MESSAGE_EXPIRY,
123
- DEFAULT_MAX_QUEUE_SIZE,
124
- MQTT_INFLIGHT_ID_LIMIT
125
- } from './constants.js';
126
-
127
- /**
128
- * Pre-flight reject threshold on store pressure.
129
- * 0.9 = 10% headroom for in-flight drain on shutdown, plus a clean band
130
- * between the yellow-health threshold (0.66, shared with the yield level the
131
- * ADR-020 Draft proposes) and store exhaustion (1.0). At or above this,
132
- * `publishNow` returns `STORAGE_FULL` synchronously; the message is not buffered.
133
- * @type {number}
134
- */
135
- const STORAGE_PRESSURE_LIMIT = 0.9;
136
-
137
- /**
138
- * Pressure threshold above which `getHealth().status` elevates to at least
139
- * 'yellow' (uniform with Terminal/QuestDB). The value matches the
140
- * pressure-aware-yield design (ADR-020, still a Draft). That design
141
- * proposes 0.66 as the pressure level where the flow would start yielding
142
- * to let sinks drain; a benchmark still has to confirm the number. Today
143
- * the yield trigger is time-only (ADR-024), so this alignment is
144
- * forward-looking.
145
- * @type {number}
146
- */
147
- const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
148
-
149
- /**
150
- * Singleton success result reused on every successful publish. Hot-path zero
151
- * allocation per ADR-013 / ADR-004. Plain literal — not frozen.
152
- * @type {{ok: true}}
153
- */
154
- const RESULT_OK = { ok: true };
155
-
156
- /**
157
- * Singleton error result for the pre-flight pressure-limit reject. Reused on
158
- * every occurrence; rare in healthy operation (pressure this close to the
159
- * limit means delivery has stopped draining), but a defined contract path.
160
- * @type {{ok: false, error: {code: string, message: string}}}
161
- */
162
- const ERR_STORAGE_FULL = {
163
- ok: false,
164
- error: {
165
- code: 'STORAGE_FULL',
166
- message: 'Store at or above pressure limit (0.9) — cannot accept message'
167
- }
168
- };
169
-
170
- /**
171
- * Singleton error result for publishes attempted while shutdown is in progress.
172
- * @type {{ok: false, error: {code: string, message: string}}}
173
- */
174
- const ERR_SHUTTING_DOWN = {
175
- ok: false,
176
- error: {
177
- code: 'SHUTTING_DOWN',
178
- message: 'Emitter is shutting down — message dropped'
179
- }
180
- };
181
-
182
- /**
183
- * Build a config-error TypeError tagged for ADR-018 fail-fast setup routing.
184
- * @param {string} message
185
- * @returns {TypeError}
186
- */
187
- const invalidConfig = function ( message ) {
188
- const err = new TypeError( message );
189
- err.code = 'INVALID_CONFIG';
190
- return err;
191
- };
192
-
193
- /**
194
- * Throw classified when an optional callback option is set to a non-function.
195
- * @param {*} value - the configured option value
196
- * @param {string} name - the option name for the error message
197
- */
198
- const assertOptionalCallback = function ( value, name ) {
199
- if ( value !== undefined && typeof value !== 'function' ) {
200
- throw invalidConfig( `${name} must be a function` );
201
- }
202
- };
203
-
204
- /**
205
- * Validate and build the mqtt.js will (last-will testament) options.
206
- *
207
- * Direct createEmitter callers bypass the DSL schema, so the factory
208
- * checks the will shape itself, and the payload encoding runs behind
209
- * the same classified-throw rule as every other setup failure
210
- * (ADR-018, fail-fast setup) — a raw codec TypeError would reach the
211
- * caller unclassified.
212
- *
213
- * @param {Object} will - config.will as supplied
214
- * @param {Object} codec - config.codec (pack + contentType)
215
- * @returns {Object} the mqtt.js connect-options `will` object
216
- */
217
- const buildWillOptions = function ( will, codec ) {
218
- if ( typeof will.topic !== 'string' || will.topic.length === 0 ) {
219
- throw invalidConfig( 'will.topic is required — a non-empty string' );
220
- }
221
- if ( will.message === undefined || will.message === null ) {
222
- throw invalidConfig( 'will.message is required' );
223
- }
224
- let willPayload;
225
- try {
226
- willPayload = Buffer.from( codec.pack( will.message ) );
227
- } catch ( packErr ) {
228
- throw invalidConfig( `will.message could not be encoded by the codec: ${packErr.message}` );
229
- }
230
- const willOptions = {
231
- topic: will.topic,
232
- payload: willPayload,
233
- // `??`, not `||`: 0 is a valid QoS.
234
- qos: will.qos ?? QOS,
235
- retain: will.retain !== false, // Default true
236
- properties: {
237
- contentType: codec.contentType
238
- }
239
- };
240
- if ( codec.payloadFormatIndicator === 1 ) {
241
- willOptions.properties.payloadFormatIndicator = true;
242
- }
243
- return willOptions;
244
- };
245
-
246
- /**
247
- * Bounded wait for the client's first 'connect' event (the broker's
248
- * connection acknowledgment). Resolves when the event fires or when
249
- * graceMs elapses, whichever comes first. Expiry is NOT an error —
250
- * the startup posture stays 'recovering' (ADR-018): the caller gets
251
- * a working handle that reports `connected: false` while mqtt.js
252
- * retries in the background. The one-shot listener is removed on
253
- * expiry so a late connack reaches only the permanent state handler;
254
- * the timer is cleared on connect so nothing pins the event loop.
255
- *
256
- * @param {Object} client - mqtt.js client (EventEmitter surface)
257
- * @param {number} graceMs - positive wait budget, milliseconds
258
- * @returns {Promise<void>} resolves on first connect or expiry; never rejects
259
- */
260
- const waitForFirstConnect = function ( client, graceMs ) {
261
- return new Promise( ( resolve ) => {
262
- let timer = null;
263
- const onConnect = function () {
264
- clearTimeout( timer );
265
- resolve();
266
- };
267
- timer = setTimeout( () => {
268
- client.removeListener( 'connect', onConnect );
269
- resolve();
270
- }, graceMs );
271
- client.once( 'connect', onConnect );
272
- } );
273
- }; // waitForFirstConnect()
167
+ import { resolveConfig } from './resolve-config.js';
168
+ import { openClient, waitForFirstConnect, attachConnectionHandlers } from './connection.js';
169
+ import { createHealth } from './health.js';
170
+ import { createPublishNow } from './publish.js';
171
+ import { createShutdown } from './shutdown-drain.js';
274
172
 
275
173
  /**
276
174
  * Create production MQTT emitter.
277
175
  *
278
- * Validation is synchronous — bad config throws before any side
279
- * effect. The return shape depends on the effective grace:
280
- * `connectGraceMs === 0` returns the handle directly (today's callers
281
- * that need synchronous creation pass 0); otherwise returns a Promise
282
- * resolving to the handle after the first connack or the grace budget.
283
- * The wiring layer awaits either shape (ADR-018).
176
+ * Validation runs before any side effect. Bad config rejects the
177
+ * returned promise with a classified `INVALID_CONFIG` error, so a
178
+ * refusal and a handle arrive through the same channel, the way
179
+ * QuestDB's factory works. With a positive grace the promise resolves
180
+ * after the first connack or the grace budget; at grace 0 it resolves
181
+ * without a wait. The wiring layer awaits it (ADR-018).
284
182
  *
285
183
  * @param {Object} config - emitter config (see configSchema in index.js)
286
- * @returns {Object|Promise<Object>} the emitter handle, or a Promise of it
184
+ * @returns {Promise<Object>} the emitter handle
287
185
  */
288
- export const createEmitter = function ( config ) {
289
- // Per ADR-018, setup-time throws carry classified err.code.
290
- //
291
- // brokerUrl: `??` so explicit '' is the user's choice (not "fall back
292
- // to env"); `.trim()` rejects whitespace-only. Symmetric with QuestDB,
293
- // which throws on `ilpUrl: ''`.
294
- const rawBrokerUrl = config.brokerUrl ?? ENV_VARS.mqttBrokerUrl;
295
- const brokerUrl = typeof rawBrokerUrl === 'string' ? rawBrokerUrl.trim() : '';
296
- if ( !brokerUrl ) {
297
- throw invalidConfig( 'brokerUrl required — set a non-empty string in .emitter() config or MQTT_BROKER_URL env var' );
298
- }
299
- if ( !config.codec ) {
300
- throw invalidConfig( 'config.codec is required' );
301
- }
302
- if ( typeof config.codec.pack !== 'function' ) {
303
- throw invalidConfig( 'config.codec must have a pack() function' );
304
- }
305
- assertOptionalCallback( config.onDeliveryFailure, 'onDeliveryFailure' );
306
- assertOptionalCallback( config.onCritical, 'onCritical' );
307
- assertOptionalCallback( config.onBackpressure, 'onBackpressure' );
308
-
309
- // First-connack grace: explicit config → MQTT_CONNECT_GRACE_MS env
310
- // fallback → 500 ms default (ADR-018 precedence; the fallback lives
311
- // here in the factory body, never as a schema sigil). `??` keeps an
312
- // explicit 0 — "hand the handle back immediately" — from falling
313
- // through to the env value. Number.isInteger also rejects Infinity,
314
- // so the wait is bounded by construction. Validated here, before
315
- // mqtt.connect opens anything (fail-fast setup per ADR-018).
316
- const connectGraceMs = config.connectGraceMs ?? ENV_VARS.mqttConnectGraceMs;
317
- if ( !Number.isInteger( connectGraceMs ) || connectGraceMs < 0 ) {
318
- throw invalidConfig( 'connectGraceMs must be a non-negative integer (milliseconds); 0 disables the first-connect wait' );
319
- }
186
+ export const createEmitter = async function ( config ) {
187
+ // Per ADR-018, setup-time refusals carry a classified err.code and
188
+ // run before any side effect. `resolveConfig` throws in one fixed
189
+ // order, and this async function turns each throw into a rejection.
190
+ const resolved = resolveConfig( config );
320
191
 
321
192
  // Core state. `unacked` is THE counter (ADR-021): +1 on accepted
322
193
  // publish, -1 when the publish callback fires. Pressure, refusal,
@@ -335,446 +206,50 @@ export const createEmitter = function ( config ) {
335
206
  }
336
207
  };
337
208
 
338
- // Broker URLs may carry credentials (mqtt://user:pass@host). Debug
339
- // logs print this redacted form, so a debug flag flipped on in
340
- // production cannot leak the password into whatever collects stdout.
341
- const redactedBrokerUrl = brokerUrl.replace( /\/\/[^@/]*@/, '//***@' );
342
-
343
- // Generate unique client ID if not provided
344
- const clientId = config.clientId || `wink-${Date.now()}-${Math.random().toString( 36 ).slice( 2, 9 )}`;
345
-
346
- // The unacked cap. Clamped to the 16-bit packet-id space: every
347
- // unacknowledged QoS-1 message holds a packet id, so one connection
348
- // can never carry more than 65,535; 60,000 leaves working headroom
349
- // (same clamp the LevelDB store applied, kept with the same warning).
350
- const requestedQueueSize = config.maxQueueSize || DEFAULT_MAX_QUEUE_SIZE;
351
- if ( requestedQueueSize > MQTT_INFLIGHT_ID_LIMIT ) {
352
- console.warn(
353
- `WinkComposer/mqtt-emitter: maxQueueSize ${requestedQueueSize} exceeds the MQTT ` +
354
- `packet-id ceiling — clamped to ${MQTT_INFLIGHT_ID_LIMIT}`
355
- );
356
- }
357
- const maxQueueSize = Math.min( requestedQueueSize, MQTT_INFLIGHT_ID_LIMIT );
358
-
359
- // Build MQTT options.
360
- //
361
- // NO outgoingStore: the client runs its default SYNCHRONOUS memory
362
- // store (ADR-021). mqtt.js erases its packet-id bookkeeping on every
363
- // connack and rebuilds it from a store snapshot asynchronously; with
364
- // an asynchronous store, writes still in flight at that instant are
365
- // invisible to the snapshot and get overwritten by id reuse — real
366
- // QoS-1 loss, measured scaling with store write latency. The
367
- // synchronous store makes the gap zero-width (validated 2026-07-09,
368
- // nine-run matrix incl. reconnects at 45k in flight).
369
- //
370
- // messageIdProvider: the client's default provider cycles the 16-bit
371
- // id space with NO in-use check — at 14 k msg/s it wraps every ~5 s,
372
- // and an id whose PUBACK never arrived gets reassigned, overwriting
373
- // the unacked packet's memory-store entry (same key). Measured at
374
- // about one lost publish per 700,000 under sustained load.
375
- // UniqueMessageIdProvider never reissues an in-use id.
376
- const mqttOptions = {
377
- ...MQTT_CONFIG,
378
- clientId,
379
- messageIdProvider: new mqtt.UniqueMessageIdProvider()
380
- };
381
-
382
- // Only add will if explicitly provided (validated + encoded by the
383
- // helper above; classified throws per ADR-018)
384
- if ( config.will ) {
385
- mqttOptions.will = buildWillOptions( config.will, config.codec );
386
- }
387
-
388
- // Create MQTT client (injectable for testing). mqtt.connect throws
389
- // its own raw TypeError on a malformed url; classify it like every
390
- // other setup failure. The error text uses the redacted url — a
391
- // malformed url can still carry credentials.
392
- const connect = config.mqttConnectFn || mqtt.connect;
393
- let client;
394
- try {
395
- client = connect( brokerUrl, mqttOptions );
396
- } catch ( connectErr ) {
397
- throw invalidConfig( `brokerUrl rejected by the MQTT client ('${redactedBrokerUrl}'): ${connectErr.message}` );
398
- }
399
-
400
- /**
401
- * Fill ratio of the unacked window: unacked / maxQueueSize, capped
402
- * at 1. Sync, O(1), allocation-free (ADR-018).
403
- * @returns {number}
404
- */
405
- const getPressure = function () {
406
- const ratio = state.unacked / maxQueueSize;
407
- return ratio > 1 ? 1 : ratio;
408
- };
409
-
410
- /**
411
- * Check and signal backpressure
412
- */
413
- const checkBackpressure = function () {
414
- const pressure = getPressure();
415
-
416
- if ( ( pressure > QUEUE_CRITICAL_THRESHOLD ) && config.onCritical ) {
417
- config.onCritical( 'QUEUE_CRITICAL', pressure );
418
- }
419
-
420
- if ( config.onBackpressure ) {
421
- config.onBackpressure( pressure );
422
- }
423
- };
424
-
425
- /**
426
- * Publish message immediately (fire-and-forget).
427
- *
428
- * Sync return per the ADR-018 sink contract:
429
- * - `{ ok: true }` on successful buffer.
430
- * - `{ ok: false, error: { code: 'STORAGE_FULL', message } }` if pre-flight
431
- * pressure check rejects (pressure >= STORAGE_PRESSURE_LIMIT).
432
- * - `{ ok: false, error: { code: 'SHUTTING_DOWN', message } }` if called
433
- * while shutdown is in progress.
434
- *
435
- * Async store-write or publish failures (after the message was buffered)
436
- * route through `onDeliveryFailure(err, { topic })` when supplied; without
437
- * a handler the adapter surfaces the failure via `Promise.reject`,
438
- * which Node logs as unhandledRejection (and v15+ exits on). Loud failure
439
- * beats silent loss.
440
- *
441
- * @param {string} topic - MQTT topic
442
- * @param {*} message - Message payload (will be encoded via config.codec)
443
- * @param {Object} [options] - Per-message options
444
- * @param {string} [options.type] - Message-type key for MESSAGE_EXPIRY lookup
445
- * @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
446
- */
447
- const publishNow = function ( topic, message, options ) {
448
- if ( state.shuttingDown ) {
449
- return ERR_SHUTTING_DOWN;
450
- }
451
-
452
- // Pre-flight pressure check — reject above STORAGE_PRESSURE_LIMIT to
453
- // leave headroom for in-flight drain and to surface backpressure to
454
- // producers before the store is literally full.
455
- if ( getPressure() >= STORAGE_PRESSURE_LIMIT ) {
456
- return ERR_STORAGE_FULL;
457
- }
458
-
459
- // Encode BEFORE the counter moves. A message the codec cannot
460
- // encode was never in flight, so it must not occupy a slot: a
461
- // leaked slot never drains, pressure ratchets up, and the
462
- // emitter ends up refusing everything. The
463
- // refusal is synchronous and classified; building it allocates,
464
- // which is fine on an error path this rare.
465
- let payload;
466
- try {
467
- payload = config.codec.pack( message );
468
- } catch ( packErr ) {
469
- state.stats.encodeErrors += 1;
470
- return {
471
- ok: false,
472
- error: {
473
- code: 'ENCODE_ERROR',
474
- message: `Message could not be encoded (topic=${topic}): ${packErr.message}`
475
- }
476
- };
477
- }
478
-
479
- const dedupId = crypto.randomUUID();
480
- const messageType = ( options && options.type ) || 'default';
481
- const expiry = MESSAGE_EXPIRY[ messageType ] || MESSAGE_EXPIRY.default;
482
-
483
- // Per-message allocations from here to the publish call (the
484
- // uuid, the properties object, the callback closure) cannot be
485
- // pre-allocated and reused: mqtt.js keeps the packet — properties
486
- // included — until PUBACK so it can retransmit, and a shared
487
- // mutated object would corrupt every queued retransmission (the
488
- // unavoidable-residual justification ADR-018 asks for).
489
- const properties = {
490
- messageExpiryInterval: expiry,
491
- userProperties: {
492
- [ WINK_NAMESPACE.dedupId ]: dedupId,
493
- [ WINK_NAMESPACE.timestamp ]: Date.now().toString(),
494
- [ WINK_NAMESPACE.version ]: '1.0'
495
- },
496
- contentType: config.codec.contentType
497
- };
209
+ const client = openClient( resolved );
498
210
 
499
- if ( config.codec.payloadFormatIndicator === 1 ) {
500
- properties.payloadFormatIndicator = true;
501
- }
502
-
503
- // Accept: the message is now in flight. The counter rises HERE,
504
- // synchronously with the accept decision, so the very next
505
- // publishNow call sees the true pressure (the old design's
506
- // optimistic-increment lesson, kept).
507
- state.unacked += 1;
508
-
509
- // Fire-and-forget publish with QoS 1. Outcome of this async callback is
510
- // observability-only — the publishNow caller has already received a
511
- // synchronous result. The callback fires exactly once per publish
512
- // (acknowledgment or failure), so the decrement below is the
513
- // counter's single exit — paired with the catch, which releases
514
- // the slot when the client rejects the call before accepting it.
515
- try {
516
- client.publish(
517
- topic,
518
- payload,
519
- { qos: QOS, properties },
520
- ( err ) => {
521
- state.unacked -= 1;
522
- if ( err ) {
523
- state.stats.publishErrors += 1;
524
- const deliveryErr = new Error(
525
- `MQTT publish failed (topic=${topic}): ${err.message || err.code || 'unknown'}`
526
- );
527
- deliveryErr.code = 'DELIVERY_FAILED';
528
- deliveryErr.cause = err;
529
- if ( config.onDeliveryFailure ) {
530
- config.onDeliveryFailure( deliveryErr, { topic } );
531
- } else {
532
- // No handler — surface as unhandledRejection so the
533
- // process logs loudly and (Node 15+) exits. Mirrors
534
- // QuestDB's persist-plan.js catch-and-throw: both
535
- // adapters route undeclared async failures
536
- // through the same Node-level escape hatch.
537
- Promise.reject( deliveryErr );
538
- }
539
- } else {
540
- state.stats.published += 1;
541
- }
542
- checkBackpressure();
543
- }
544
- );
545
- } catch ( publishErr ) {
546
- // The client threw before accepting the message; its callback
547
- // will never fire, so the slot is released here — the sync
548
- // face of DELIVERY_FAILED (see the header vocabulary).
549
- state.unacked -= 1;
550
- state.stats.publishErrors += 1;
551
- return {
552
- ok: false,
553
- error: {
554
- code: 'DELIVERY_FAILED',
555
- message: `MQTT publish rejected (topic=${topic}): ${publishErr.message}`
556
- }
557
- };
558
- }
559
-
560
- return RESULT_OK;
561
- };
562
-
563
- // Shutdown outcome, latched on the first call. A second
564
- // caller used to get an instant clean resolve — mid-drain, or even
565
- // after a shutdown that recorded dropped messages. Every caller now
566
- // receives the first call's promise.
567
- let shutdownPromise = null;
568
-
569
- const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5000;
570
-
571
- /**
572
- * The real shutdown body. `shutdown` below latches its promise so
573
- * every caller receives this one outcome.
574
- *
575
- * **Drain-then-close** (a sustained-load finding, kept through the
576
- * ADR-021 rework): mqtt.js's own `end()` does not reliably drain
577
- * in-flight publishes, so the emitter waits on its own counter.
578
- *
579
- * 1. Sets `state.shuttingDown` so `publishNow` rejects new work
580
- * (returns `SHUTTING_DOWN` immediately).
581
- * 2. **Polls the unacked counter until zero**, using the whole
582
- * `timeout` budget — no early give-up. The wait runs whether or
583
- * not the connection is up at the moment of the call: mqtt.js
584
- * auto-reconnects, so a mid-drain reconnect can still deliver
585
- * the backlog inside the budget.
586
- * 3. Once drained (or budget expired), calls `client.end(...)`
587
- * with the remaining time as the broker-DISCONNECT budget.
588
- * 4. Force-closes if the broker hangs the DISCONNECT.
589
- *
590
- * The old design's shutdown re-drive (re-sending stragglers through
591
- * a store stream scan) died with the disk store: with the memory
592
- * store there is no store to scan, and a message whose PUBACK was
593
- * lost in-session is re-sent by the client itself on the next
594
- * reconnect. If it is still unacknowledged at the deadline, the
595
- * loss report counts it.
596
- *
597
- * A clean resolve is a delivery statement (ADR-018). When the
598
- * drain could not complete, shutdown rejects with classified
599
- * `SHUTDOWN_TIMEOUT` and `dropped: { count }` — the counter's exact
600
- * value — after the close has already happened (teardown first,
601
- * then the report). This fires CONNECTED OR NOT: with no disk
602
- * store, nothing survives the process, so a disconnected shutdown
603
- * with pending messages is a real loss and is reported as one. (The
604
- * wal-backed design resolved that case clean on purpose — its disk
605
- * store held the messages for the next session. ADR-021 traded that
606
- * away and says so.)
607
- */
608
- const doShutdown = async function ( timeout ) {
609
- state.shuttingDown = true;
610
-
611
- // A non-finite or non-positive timeout collapses the drain
612
- // (Infinity overflows setTimeout, which clamps it to ~1 ms;
613
- // NaN fails every deadline comparison), so it falls back to
614
- // the default budget.
615
- const budget = ( Number.isFinite( timeout ) && timeout > 0 ) ? timeout : DEFAULT_SHUTDOWN_TIMEOUT_MS;
616
- const deadline = Date.now() + budget;
617
- const drainPollIntervalMs = 25;
618
-
619
- // Drain wait — one cheap counter read per tick; sequential
620
- // awaits are the wait-for-condition pattern.
621
- while ( Date.now() < deadline && state.unacked > 0 ) {
622
- // eslint-disable-next-line no-await-in-loop
623
- await new Promise( ( r ) => setTimeout( r, drainPollIntervalMs ) );
624
- }
625
-
626
- // Whatever is still unacknowledged goes undelivered — publishNow
627
- // stopped accepting work at step 1, so the counter has settled
628
- // into its final value. It becomes the `dropped` payload below.
629
- const undeliveredCount = state.unacked;
630
- const remaining = Date.now() - deadline;
631
- const closeBudget = remaining < 0 ? Math.abs( remaining ) : 1;
632
-
633
- await new Promise( ( resolve ) => {
634
- const forceTimer = setTimeout( () => {
635
- client.end( true );
636
- resolve();
637
- }, closeBudget );
638
-
639
- if ( state.connected ) {
640
- client.end( false, {}, () => {
641
- clearTimeout( forceTimer );
642
- resolve();
643
- } );
644
- } else {
645
- clearTimeout( forceTimer );
646
- client.end( true );
647
- resolve();
648
- }
649
- } );
650
-
651
- // Teardown first, then the report: the close has already happened;
652
- // an incomplete drain must not read as a clean one (ADR-018).
653
- if ( undeliveredCount > 0 ) {
654
- const err = new Error(
655
- `WinkComposer/mqtt-emitter: shutdown closed with ${undeliveredCount} message(s) unacknowledged`
656
- );
657
- err.code = 'SHUTDOWN_TIMEOUT';
658
- err.dropped = { count: undeliveredCount };
659
- throw err;
660
- }
661
- return undefined;
662
- }; // doShutdown()
663
-
664
- /**
665
- * Graceful shutdown — drain semantics with a timeout floor.
666
- *
667
- * Accepts the ADR-018 shutdown-contract shape `{ timeout }` (defaults to
668
- * 5000ms). The destructure with `= {}` lets callers invoke
669
- * `shutdown()` with no argument, `shutdown( {} )`, or
670
- * `shutdown( { timeout: N } )`.
671
- *
672
- * The outcome is latched: the first call runs the shutdown, every
673
- * later call returns the same promise. A caller
674
- * arriving mid-drain waits for the real teardown; a caller arriving
675
- * after a lossy shutdown sees the same classified rejection, never
676
- * a contradicting clean resolve. One consequence: the first
677
- * caller's `{ timeout }` governs; a later caller's is ignored.
678
- *
679
- * @param {{timeout?: number}} [options]
680
- * @returns {Promise<void>}
681
- */
682
- const shutdown = function ( { timeout = DEFAULT_SHUTDOWN_TIMEOUT_MS } = {} ) {
683
- if ( !shutdownPromise ) {
684
- shutdownPromise = doShutdown( timeout );
685
- }
686
- return shutdownPromise;
687
- }; // shutdown()
688
-
689
- // Connection handlers - minimal, no automatic status
690
- client.on( 'connect', () => {
691
- state.connected = true;
692
- // mqtt.js fires 'connect' on every connack, including mid-run
693
- // reconnects. Counting the later ones gives operators — and the
694
- // release-soak signature policy — an observable "a reconnect
695
- // happened during this run" fact via getHealth().stats.
696
- if ( state.hasConnectedOnce ) {
697
- state.stats.reconnects += 1;
698
- }
699
- state.hasConnectedOnce = true;
700
- if ( config.debug ) {
701
- console.log( `Connected to ${redactedBrokerUrl}` );
702
- }
211
+ const { getPressure, checkCritical, checkBackpressure, getHealth } = createHealth( {
212
+ state,
213
+ maxQueueSize: resolved.maxQueueSize,
214
+ onCritical: resolved.onCritical,
215
+ onBackpressure: resolved.onBackpressure
703
216
  } );
704
-
705
- client.on( 'offline', () => {
706
- state.connected = false;
707
- if ( config.debug ) {
708
- console.log( `Offline - ${state.unacked} messages in flight` );
709
- }
217
+ const publishNow = createPublishNow( {
218
+ client,
219
+ state,
220
+ codec: resolved.codec,
221
+ getPressure,
222
+ checkCritical,
223
+ checkBackpressure,
224
+ onDeliveryFailure: resolved.onDeliveryFailure
710
225
  } );
711
-
712
- client.on( 'error', ( err ) => {
713
- state.stats.errors += 1;
714
- if ( config.debug ) {
715
- console.error( `MQTT error: ${err.message}` );
716
- }
226
+ const { flush, shutdown } = createShutdown( { client, state } );
227
+
228
+ // The permanent handlers attach before the grace wait's one-shot
229
+ // listener, so a handle resolved via connack already reports
230
+ // `connected: true`.
231
+ attachConnectionHandlers( {
232
+ client,
233
+ state,
234
+ debug: resolved.debug,
235
+ redactedBrokerUrl: resolved.redactedBrokerUrl
717
236
  } );
718
237
 
719
- /**
720
- * Health snapshot for operator monitoring (uniform across sinks).
721
- *
722
- * Returns the ADR-018 health floor `{status, connected, pressure}` plus
723
- * this adapter's own `stats` addition. `stats.unacked` is the live
724
- * counter — the number of messages
725
- * accepted but not yet acknowledged by the broker; exactly what a
726
- * process crash at this instant would cost (ADR-021).
727
- *
728
- * Status derivation (uniform with Terminal/QuestDB; see file header):
729
- * - `red` when `!connected`
730
- * - `yellow` when connected AND `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD`
731
- * - `green` otherwise
732
- *
733
- * The wal-backed design's `storeHealth`, `circuitState`, and `metrics`
734
- * diagnostics died with the LevelDB store (ADR-021).
735
- *
736
- * @returns {Object} Health snapshot — required floor + live counter
737
- */
738
- const getHealth = function () {
739
- const pressure = getPressure();
740
-
741
- let status;
742
- if ( !state.connected ) {
743
- status = 'red';
744
- } else if ( pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD ) {
745
- status = 'yellow';
746
- } else {
747
- status = 'green';
748
- }
749
-
750
- return {
751
- // Required floor (ADR-018)
752
- status,
753
- connected: state.connected,
754
- pressure,
755
- // This adapter's addition beyond the floor
756
- stats: { ...state.stats, unacked: state.unacked }
757
- };
758
- };
759
-
760
238
  const handle = {
761
239
  publishNow,
240
+ flush,
762
241
  getHealth,
763
242
  shutdown,
764
243
  getPressure
765
244
  };
766
245
 
767
- // First-connack grace (bounded). With a grace, the return is a
768
- // Promise the wiring layer awaits (ADR-018 allows sync or async
769
- // factories): by the time wire() hands the emitter to the flow,
770
- // the client has either seen its first connack or spent the
771
- // budget. Expiry is not an error — the posture stays 'recovering'.
772
- // With connectGraceMs 0 the handle returns synchronously, exactly
773
- // the pre-grace behavior. The permanent 'connect' handler above is
774
- // attached before the wait's one-shot listener, so a handle
775
- // resolved via connack already reports `connected: true`.
776
- if ( connectGraceMs === 0 ) {
777
- return handle;
246
+ // First-connack grace (bounded). By the time wire() hands the
247
+ // emitter to the flow, the client has either seen its first
248
+ // connack or spent the budget. Expiry is not an error — the
249
+ // posture stays 'recovering'. With connectGraceMs 0 no wait is
250
+ // armed and the promise resolves at once.
251
+ if ( resolved.connectGraceMs > 0 ) {
252
+ await waitForFirstConnect( client, resolved.connectGraceMs );
778
253
  }
779
- return waitForFirstConnect( client, connectGraceMs ).then( () => handle );
780
- }; // createMQTTEmitter()
254
+ return handle;
255
+ }; // createEmitter()