@winkjs/composer 0.6.0 → 0.7.0

package/src/core/emitter-manager/mqtt/health.js

@@ -0,0 +1,151 @@
+// core/emitter-manager/mqtt/health.js
+
+/**
+ * @fileoverview Pressure and health of the MQTT emitter (ADR-018 §8,
+ * ADR-021).
+ *
+ * One counter, `state.unacked`, feeds everything here. `getPressure()`
+ * is that counter over the cap, and `getHealth()` derives the status
+ * from the link and the pressure. The factory builds this once;
+ * nothing allocates on the pressure read.
+ *
+ * Two pressure callbacks live here, with different cadences.
+ * `onBackpressure` is a level signal: `checkBackpressure()` calls it
+ * after every acknowledgment with the current pressure. `onCritical`
+ * is an edge signal: `checkCritical()` calls it once when an accepted
+ * publish lifts pressure above the critical threshold, then disarms
+ * it. An acknowledgment that brings pressure below the yellow
+ * threshold re-arms it. Pressure rises only on accepts, so the
+ * crossing is detected on the accept path. During a broker outage no
+ * acknowledgment arrives, and an ack-side check would stay silent
+ * through the whole climb. The accept-path cost is one boolean read
+ * and, while armed, one comparison. A level signal here would repeat
+ * the same warning on every acknowledgment while pressure stays high,
+ * and the message-level callback guard prints each call it catches.
+ *
+ * Health-status semantics (uniform across sinks):
+ * - `red`    if `!state.connected`.
+ * - `yellow` if connected AND `pressure >= 0.66` (threshold shared with
+ *   ADR-020's Draft yield proposal).
+ * - `green`  otherwise.
+ *
+ * @see ADR-018
+ * @see ADR-021
+ */
+
+import { QUEUE_CRITICAL_THRESHOLD } from './constants.js';
+
+/**
+ * Pressure threshold above which `getHealth().status` elevates to at least
+ * 'yellow' (uniform with Terminal/QuestDB). The value matches the
+ * pressure-aware-yield design (ADR-020, still a Draft). That design
+ * proposes 0.66 as the pressure level where the flow would start yielding
+ * to let sinks drain; a benchmark still has to confirm the number. Today
+ * the yield trigger is time-only (ADR-024), so this alignment is
+ * forward-looking.
+ * @type {number}
+ */
+const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
+
+/**
+ * Builds the pressure and health readers over the shared state.
+ *
+ * @param {Object} deps
+ * @param {Object} deps.state - The emitter's core state
+ * @param {number} deps.maxQueueSize - The unacked cap
+ * @param {function|null} deps.onCritical - Guard-wrapped high-pressure callback, or null
+ * @param {function|null} deps.onBackpressure - Guard-wrapped pressure callback, or null
+ * @returns {{getPressure: function, checkCritical: function, checkBackpressure: function, getHealth: function}}
+ */
+export const createHealth = function ( { state, maxQueueSize, onCritical, onBackpressure } ) {
+    // The edge state of `onCritical`. Armed means the next climb past
+    // the critical threshold fires the callback. It starts armed.
+    let criticalArmed = true;
+
+    /**
+     * Fill ratio of the unacked window: unacked / maxQueueSize, capped
+     * at 1. Sync, O(1), allocation-free (ADR-018).
+     * @returns {number}
+     */
+    const getPressure = function () {
+        const ratio = state.unacked / maxQueueSize;
+        return ratio > 1 ? 1 : ratio;
+    };
+
+    /**
+     * Fires `onCritical` once when an accepted publish lifts pressure
+     * above the critical threshold, then disarms it. Runs on the
+     * accept path, after the counter rises. The disarm comes before
+     * the call, so a callback that throws (contained by the guard)
+     * still counts as fired.
+     */
+    const checkCritical = function () {
+        if ( criticalArmed && onCritical ) {
+            const pressure = getPressure();
+            if ( pressure > QUEUE_CRITICAL_THRESHOLD ) {
+                criticalArmed = false;
+                onCritical( 'QUEUE_CRITICAL', pressure );
+            }
+        }
+    };
+
+    /**
+     * Runs after every acknowledgment. Re-arms `onCritical` once
+     * pressure is back below the yellow threshold, and reports the
+     * current pressure to `onBackpressure`.
+     */
+    const checkBackpressure = function () {
+        const pressure = getPressure();
+
+        if ( pressure < HEALTH_PRESSURE_YELLOW_THRESHOLD ) {
+            criticalArmed = true;
+        }
+
+        if ( onBackpressure ) {
+            onBackpressure( pressure );
+        }
+    };
+
+    /**
+     * Health snapshot for operator monitoring (uniform across sinks).
+     *
+     * Returns the ADR-018 health floor `{status, connected, pressure}` plus
+     * this adapter's own `stats` addition. `stats.unacked` is the live
+     * counter — the number of messages
+     * accepted but not yet acknowledged by the broker; exactly what a
+     * process crash at this instant would cost (ADR-021).
+     *
+     * Status derivation (uniform with Terminal/QuestDB; see file header):
+     * - `red`    when `!connected`
+     * - `yellow` when connected AND `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD`
+     * - `green`  otherwise
+     *
+     * The wal-backed design's `storeHealth`, `circuitState`, and `metrics`
+     * diagnostics died with the LevelDB store (ADR-021).
+     *
+     * @returns {Object} Health snapshot — required floor + live counter
+     */
+    const getHealth = function () {
+        const pressure = getPressure();
+
+        let status;
+        if ( !state.connected ) {
+            status = 'red';
+        } else if ( pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD ) {
+            status = 'yellow';
+        } else {
+            status = 'green';
+        }
+
+        return {
+            // Required floor (ADR-018)
+            status,
+            connected: state.connected,
+            pressure,
+            // This adapter's addition beyond the floor
+            stats: { ...state.stats, unacked: state.unacked }
+        };
+    };
+
+    return { getPressure, checkCritical, checkBackpressure, getHealth };
+}; // createHealth()

package/src/core/emitter-manager/mqtt/index.js

@@ -36,10 +36,55 @@
  * `id`, `configSchema`, `createEmitter`, `durabilityClass`, and the
  * default aggregate referencing the same constants.
  *
+ * Shed policy and outage budget. Once the unacknowledged count reaches
+ * 90% of `maxQueueSize`, `publishNow` refuses new messages with
+ * `STORAGE_FULL`, and the messages already accepted stay safe. With
+ * the default cap of 10,000, refusal starts at 9,000 messages. That
+ * is 15 minutes of outage at 10 messages a second, and 2.5 hours at
+ * one a second. Memory is the count times the payload size: about
+ * 10 MB at 1 KB a message.
+ *
+ * Ordering. While connected, messages reach the broker in publish
+ * order over the one connection. After a reconnect the client replays
+ * its in-memory store in insertion order, and new publishes wait in
+ * the client's processing queue until that replay ends (mqtt.js
+ * 5.15.1, `client.js:1230-1262`). Delivery is at-least-once, so a
+ * consumer can see a message twice. Every message carries a
+ * `winkDedupId`, and the MQTT source drops the repeats (ADR-022).
+ *
+ * Edge-baseline defaults (every one runs within a 2 GB device):
+ *
+ *   | Setting          | Default  | Environment variable      |
+ *   |------------------|----------|---------------------------|
+ *   | `maxQueueSize`   | 10000    | `MQTT_MAX_QUEUE_SIZE`     |
+ *   | `connectGraceMs` | 500      | `MQTT_CONNECT_GRACE_MS`   |
+ *   | message expiry   | 3600 s   | `MQTT_MSG_EXPIRY`         |
+ *   | keepalive        | 60 s     | `MQTT_KEEPALIVE`          |
+ *   | reconnect period | 5000 ms  | `MQTT_RECONNECT_MS`       |
+ *   | connect timeout  | 30000 ms | `MQTT_CONNECT_TIMEOUT_MS` |
+ *
+ *   `brokerUrl` falls back to `MQTT_BROKER_URL`. The reconnect period
+ *   gains a random share of up to 20% per client (`utils/jitter`).
+ *   The cap is clamped to 60,000, the packet-id ceiling.
+ *
+ * Related decisions beyond ADR-018 and ADR-021. ADR-022: this adapter
+ * writes the `winkDedupId` the source dedups on. ADR-023: `publishNow`
+ * reads the message during the call and keeps no reference. ADR-027:
+ * the three callbacks run through the shared guard. ADR-028: every
+ * line and error follows the message grammar. ADR-026 (Proposed) is
+ * the planned TLS surface for `mqtts://`.
+ *
  * Directory structure:
  * core/emitter-manager/mqtt/
  * ├── index.js                 # Module surface: id, schema, factory
- * ├── emitter.js               # The emitter implementation
+ * ├── emitter.js               # The factory: resolve, open, assemble
+ * ├── resolve-config.js        # Validation in one fixed order; the
+ * │                            #   three callbacks armed by the guard
+ * ├── connection.js            # Client open, first-connack wait, the
+ * │                            #   connection event handlers
+ * ├── health.js                # getPressure, getHealth, backpressure
+ * ├── publish.js               # publishNow, the hot path
+ * ├── shutdown-drain.js        # The latched drain-then-close shutdown
  * ├── mqtt-store.js            # DORMANT LevelDB store (ADR-021; kept
  * │                            #   for the WAL successor, not wired,
  * │                            #   not shipped in the npm package)
@@ -47,7 +92,9 @@
  * └── test/                    # Spec files
  *
  * @example
- * const emitter = createEmitter({
+ * // The factory is async, as QuestDB's is. A configuration error
+ * // rejects the promise with a classified `INVALID_CONFIG` error.
+ * const emitter = await createEmitter({
  *     brokerUrl: 'mqtt://broker.local:1883',
  *     codec: jsonCodec
  * });
@@ -59,8 +106,24 @@
  */
 
 import { validators } from '../../utils/validate/index.js';
+import { classifyAddress } from '../../utils/address/index.js';
 import { createEmitter } from './emitter.js';
 
+/**
+ * Schema validator for `brokerUrl`: non-empty and never `localhost`
+ * (ADR-030). A value the URL grammar cannot read passes; the MQTT
+ * library reports its own error for it.
+ *
+ * @param {*} value - The configured value
+ * @returns {boolean} Whether the value is allowed
+ */
+const isAllowedBrokerUrl = function ( value ) {
+    if ( !validators.nonEmptyString( value ) ) {
+        return false;
+    }
+    return classifyAddress( value, 'url' ).kind !== 'localhost';
+}; // isAllowedBrokerUrl()
+
 /**
  * Emitter identifier - must match target in emitIf specs.
  * @type {string}
@@ -106,8 +169,9 @@ export const configSchema = {
     brokerUrl: {
         type: 'string',
         required: false,
-        minLength: 1,
-        error: 'brokerUrl must be a non-empty string (e.g., mqtt://broker.local:1883)'
+        validator: isAllowedBrokerUrl,
+        error: 'brokerUrl must be a non-empty broker URL with a literal address or a name, never ' +
+            'localhost; e.g., mqtt://127.0.0.1:1883'
     },
     codec: {
         type: 'object',

package/src/core/emitter-manager/mqtt/publish.js

@@ -0,0 +1,292 @@
+// core/emitter-manager/mqtt/publish.js
+
+/**
+ * @fileoverview The MQTT emitter's hot path: `publishNow` (ADR-013,
+ * ADR-018, ADR-021).
+ *
+ * A publish is fire-and-forget with QoS 1. The call encodes the
+ * message, checks pressure, raises the unacked counter, and hands the
+ * packet to mqtt.js. It returns a synchronous `{ ok }` result and
+ * never a promise. The client's callback fires once per packet, on
+ * acknowledgment or failure, and `settlePublish` lowers the counter
+ * there. So the counter's single exit is that callback, paired with
+ * the catch that releases the slot when the client rejects the call
+ * before accepting it.
+ *
+ * Refusals are classified, synchronous, and allocation-free where the
+ * result can be shared: `SHUTTING_DOWN` and `STORAGE_FULL` are
+ * singletons. `ENCODE_ERROR` and the sync face of `DELIVERY_FAILED`
+ * build their result, because they carry the failure's own text and
+ * are rare.
+ *
+ * @see ADR-013
+ * @see ADR-018
+ * @see ADR-021
+ */
+
+import crypto from 'crypto';
+import {
+    QOS,
+    WINK_NAMESPACE,
+    MESSAGE_EXPIRY
+} from './constants.js';
+
+/**
+ * Pre-flight reject threshold on store pressure.
+ * 0.9 = 10% headroom for in-flight drain on shutdown, plus a clean band
+ * between the yellow-health threshold (0.66, shared with the yield level the
+ * ADR-020 Draft proposes) and store exhaustion (1.0). At or above this,
+ * `publishNow` returns `STORAGE_FULL` synchronously; the message is not buffered.
+ * @type {number}
+ */
+const STORAGE_PRESSURE_LIMIT = 0.9;
+
+/**
+ * Singleton success result reused on every successful publish. Hot-path zero
+ * allocation per ADR-013 / ADR-004. Plain literal — not frozen.
+ * @type {{ok: true}}
+ */
+const RESULT_OK = { ok: true };
+
+/**
+ * Singleton error result for the pre-flight pressure-limit reject. Reused on
+ * every occurrence; rare in healthy operation (pressure this close to the
+ * limit means delivery has stopped draining), but a defined contract path.
+ * @type {{ok: false, error: {code: string, message: string}}}
+ */
+const ERR_STORAGE_FULL = {
+    ok: false,
+    error: {
+        code: 'STORAGE_FULL',
+        message: 'Store at or above pressure limit (0.9) — cannot accept message'
+    }
+};
+
+/**
+ * Singleton error result for publishes attempted while shutdown is in progress.
+ * @type {{ok: false, error: {code: string, message: string}}}
+ */
+const ERR_SHUTTING_DOWN = {
+    ok: false,
+    error: {
+        code: 'SHUTTING_DOWN',
+        message: 'Emitter is shutting down — message dropped'
+    }
+};
+
+/**
+ * The `ENCODE_ERROR` refusal. Built per event, because it carries the
+ * codec's own message; the path is rare.
+ *
+ * @param {string} topic - The topic of the refused message
+ * @param {Error} packErr - The codec's throw
+ * @returns {{ok: false, error: {code: string, message: string}}}
+ */
+const encodeRefusal = function ( topic, packErr ) {
+    return {
+        ok: false,
+        error: {
+            code: 'ENCODE_ERROR',
+            message: `Message could not be encoded (topic=${topic}): ${packErr.message}`
+        }
+    };
+}; // encodeRefusal()
+
+/**
+ * The sync face of `DELIVERY_FAILED`: the client threw before
+ * accepting the message, so nothing is in flight.
+ *
+ * @param {string} topic - The topic of the refused message
+ * @param {Error} publishErr - The client's throw
+ * @returns {{ok: false, error: {code: string, message: string}}}
+ */
+const publishRefusal = function ( topic, publishErr ) {
+    return {
+        ok: false,
+        error: {
+            code: 'DELIVERY_FAILED',
+            message: `MQTT publish rejected (topic=${topic}): ${publishErr.message}`
+        }
+    };
+}; // publishRefusal()
+
+/**
+ * Builds the MQTT v5 properties of one publish: the expiry, the
+ * dedup id, the timestamp, and the content type.
+ *
+ * Per-message allocations here (the uuid, the properties object) and
+ * the callback closure in `publishNow` cannot be pre-allocated and
+ * reused: mqtt.js keeps the packet — properties included — until
+ * PUBACK so it can retransmit, and a shared mutated object would
+ * corrupt every queued retransmission (the unavoidable-residual
+ * justification ADR-018 asks for). The options object that carries
+ * `properties` into the client is not on this list; the client
+ * copies it at once, so one object per emitter serves every publish.
+ *
+ * @param {Object} codec - The configured codec (contentType, payloadFormatIndicator)
+ * @param {number} expiry - The message expiry in seconds
+ * @returns {Object} the properties object mqtt.js keeps until PUBACK
+ */
+const buildProperties = function ( codec, expiry ) {
+    const properties = {
+        messageExpiryInterval: expiry,
+        userProperties: {
+            [ WINK_NAMESPACE.dedupId ]: crypto.randomUUID(),
+            [ WINK_NAMESPACE.timestamp ]: Date.now().toString(),
+            [ WINK_NAMESPACE.version ]: '1.0'
+        },
+        contentType: codec.contentType
+    };
+
+    if ( codec.payloadFormatIndicator === 1 ) {
+        properties.payloadFormatIndicator = true;
+    }
+    return properties;
+}; // buildProperties()
+
+/**
+ * Builds `publishNow` over the shared state and the armed callbacks.
+ *
+ * @param {Object} deps
+ * @param {Object} deps.client - mqtt.js client
+ * @param {Object} deps.state - The emitter's core state
+ * @param {Object} deps.codec - The configured codec (pack, contentType)
+ * @param {function} deps.getPressure - The pressure reader from health.js
+ * @param {function} deps.checkCritical - The accept-path edge check from health.js
+ * @param {function} deps.checkBackpressure - The ack-path pressure signaller from health.js
+ * @param {function|null} deps.onDeliveryFailure - Guard-wrapped delivery-failure callback, or null
+ * @returns {function} publishNow
+ */
+export const createPublishNow = function ( {
+    client, state, codec, getPressure, checkCritical, checkBackpressure, onDeliveryFailure
+} ) {
+    // One options object per emitter, reused on every publish. mqtt.js
+    // copies its options synchronously before any other work and keeps
+    // only `properties` in the packet (5.15.1, `client.js:376-381`),
+    // so the shared object is never retained. `properties` is assigned
+    // per call; it is the object the client keeps until PUBACK.
+    const publishOptions = { qos: QOS, properties: null };
+
+    /**
+     * Settles one publish when its callback fires. Outcome of this async
+     * callback is observability-only — the publishNow caller has
+     * already received a synchronous result. The callback fires exactly
+     * once per publish (acknowledgment or failure), so the decrement
+     * here is the counter's single exit.
+     *
+     * @param {Error|null} err - The client's failure, or null on acknowledgment
+     * @param {string} topic - The published topic, for the failure report
+     */
+    const settlePublish = function ( err, topic ) {
+        state.unacked -= 1;
+        if ( err ) {
+            state.stats.publishErrors += 1;
+            const deliveryErr = new Error(
+                `winkComposer/mqttEmitter: publish failed (topic=${topic}): ${err.message || err.code || 'unknown'}`
+            );
+            deliveryErr.code = 'DELIVERY_FAILED';
+            deliveryErr.cause = err;
+            if ( onDeliveryFailure ) {
+                onDeliveryFailure( deliveryErr, { topic } );
+            } else {
+                // No handler — surface as unhandledRejection so the
+                // process logs loudly and (Node 15+) exits. Mirrors
+                // QuestDB's persist-plan.js catch-and-throw: both
+                // adapters route undeclared async failures
+                // through the same Node-level escape hatch.
+                Promise.reject( deliveryErr );
+            }
+        } else {
+            state.stats.published += 1;
+        }
+        checkBackpressure();
+    }; // settlePublish()
+
+    /**
+     * Publish message immediately (fire-and-forget).
+     *
+     * Sync return per the ADR-018 sink contract:
+     * - `{ ok: true }` on successful buffer.
+     * - `{ ok: false, error: { code: 'STORAGE_FULL', message } }` if pre-flight
+     *   pressure check rejects (pressure >= STORAGE_PRESSURE_LIMIT).
+     * - `{ ok: false, error: { code: 'SHUTTING_DOWN', message } }` if called
+     *   while shutdown is in progress.
+     *
+     * Async store-write or publish failures (after the message was buffered)
+     * route through `onDeliveryFailure(err, { topic })` when supplied; without
+     * a handler the adapter surfaces the failure via `Promise.reject`,
+     * which Node logs as unhandledRejection (and v15+ exits on). Loud failure
+     * beats silent loss.
+     *
+     * @param {string} topic - MQTT topic
+     * @param {*} message - Message payload (will be encoded via config.codec)
+     * @param {Object} [options] - Per-message options
+     * @param {string} [options.type] - Message-type key for MESSAGE_EXPIRY lookup
+     * @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
+     */
+    const publishNow = function ( topic, message, options ) {
+        if ( state.shuttingDown ) {
+            return ERR_SHUTTING_DOWN;
+        }
+
+        // Pre-flight pressure check — reject above STORAGE_PRESSURE_LIMIT to
+        // leave headroom for in-flight drain and to surface backpressure to
+        // producers before the store is literally full.
+        if ( getPressure() >= STORAGE_PRESSURE_LIMIT ) {
+            return ERR_STORAGE_FULL;
+        }
+
+        // Encode BEFORE the counter moves. A message the codec cannot
+        // encode was never in flight, so it must not occupy a slot. A
+        // leaked slot never drains, pressure only rises, and the
+        // emitter ends up refusing everything. The refusal is
+        // synchronous and classified. Building it allocates, which is
+        // fine on an error path this rare.
+        let payload;
+        try {
+            payload = codec.pack( message );
+        } catch ( packErr ) {
+            state.stats.encodeErrors += 1;
+            return encodeRefusal( topic, packErr );
+        }
+
+        const messageType = ( options && options.type ) || 'default';
+        const expiry = MESSAGE_EXPIRY[ messageType ] || MESSAGE_EXPIRY.default;
+        publishOptions.properties = buildProperties( codec, expiry );
+
+        // Accept: the message is now in flight. The counter rises HERE,
+        // synchronously with the accept decision, so the very next
+        // publishNow call sees the true pressure (the old design's
+        // optimistic-increment lesson, kept).
+        state.unacked += 1;
+
+        // Fire-and-forget publish with QoS 1. The callback settles the
+        // counter (see settlePublish above), paired with the catch,
+        // which releases the slot when the client rejects the call
+        // before accepting it.
+        try {
+            client.publish(
+                topic,
+                payload,
+                publishOptions,
+                ( err ) => settlePublish( err, topic )
+            );
+        } catch ( publishErr ) {
+            // The client threw before accepting the message; its callback
+            // will never fire, so the slot is released here — the sync
+            // face of DELIVERY_FAILED (see the header vocabulary).
+            state.unacked -= 1;
+            state.stats.publishErrors += 1;
+            return publishRefusal( topic, publishErr );
+        }
+
+        // The message is accepted and the counter has its new value.
+        // Pressure rises only here, so this is where a climb past the
+        // critical threshold is seen (see health.js).
+        checkCritical();
+
+        return RESULT_OK;
+    }; // publishNow()
+
+    return publishNow;
+}; // createPublishNow()