@winkjs/composer 0.0.2 → 0.4.0

package/src/core/source-manager/mqtt/dedup.js

@@ -0,0 +1,244 @@
+// core/source-manager/mqtt/dedup.js
+
+/**
+ * @fileoverview Time-bounded, count-capped dedup cache for MQTT QoS 1
+ * retransmissions (ADR-022).
+ *
+ * QoS 1 is "at least once": a re-send after a connection break carries
+ * the same `winkDedupId` as its original, and this cache drops the
+ * repeat. A duplicate arrives roughly one outage after its original,
+ * so the cache is specified in TIME (catch any duplicate within
+ * `windowMs`) and guaranteed in MEMORY (never hold more than
+ * `maxEntries` ids). Whichever bound binds first wins; at high message
+ * rates the effective window is `maxEntries ÷ rate` — the trade is
+ * documented in ADR-022, not hidden.
+ *
+ * One structure, no modes, no timers:
+ * - A `Set` gives O(1) duplicate lookup.
+ * - A pre-allocated ring (id slots + a Float64Array of arrival
+ *   timestamps) gives O(1) insertion and eviction in arrival order.
+ * - Expiry is checked at the ring head on each call — amortized O(1),
+ *   no background sweep.
+ *
+ * Hot-path allocation profile (ADR-018 zero-alloc — allocation only where
+ * unavoidable, each residual stated, no overclaiming):
+ * - Our code allocates nothing per message: the ring, the timestamp
+ *   array, and the Set are created at initialization; ids are stored
+ *   by reference; no objects, arrays, or strings are built.
+ * - Two unavoidable residuals, both bounded: (1) `Set.add`/`Set.delete`
+ *   maintain V8's internal hash table — amortized O(1), memory bounded
+ *   by `maxEntries`, and the identical profile the pre-ADR-022 cache
+ *   already had (JS offers no pre-sized Set). (2) `Date.now()` exceeds
+ *   the small-integer range, so the timestamp is transiently boxed; it
+ *   unboxes on store into the Float64Array and dies in the nursery.
+ * - Eviction clears the ring slot so an evicted id string is not
+ *   retained past its window — that release is deliberate.
+ *
+ * Opt-in by construction: a message without a dedup id (null or
+ * undefined) bypasses the cache entirely — identity is never guessed
+ * from content (ADR-022 rejected content hashing: byte-identical
+ * telemetry is routinely two real readings).
+ *
+ * Crash residue (accepted, per ADR-022): the cache is in-memory, so a
+ * crash between processing and acknowledging a message means its
+ * broker redelivery after restart is processed once more. At most one
+ * duplicate per source crash; disk-backed dedup state was rejected.
+ *
+ *   ASSUMPTIONS
+ *   -----------
+ *   1. A dedup id is unique per message and repeats only on a QoS-1
+ *      retransmission (the emitter stamps UUIDs).
+ *   2. The clock moves forward at millisecond granularity. A backward
+ *      clock jump can expire entries early or hold them one window too
+ *      long — bounded either way, and self-healing within one window.
+ *
+ *   LIMITATIONS
+ *   -----------
+ *   1. In-memory: the cache dies with the process, so a duplicate
+ *      arriving just after a restart may slip through. Deliberate
+ *      (ADR-022 rejected disk-backed state); at most one per crash;
+ *      QuestDB's at-rest dedup is the backstop.
+ *   2. At high rates the count cap binds before the time window: the
+ *      effective window is maxEntries ÷ rate (the full 120 s up to
+ *      about 550 msg/s at defaults; ~6.5 s at 10 k msg/s). Stated in
+ *      ADR-022, not hidden.
+ *   3. Memory, measured (benchmark/mqtt-source/dedup-memory.js,
+ *      2026-07-09): 86 bytes per production-shaped id, ~6.7 MB worst
+ *      case at the 65,536 cap including bounded V8 Set slack.
+ *
+ * @see ADR-022 - The full decision
+ */
+
+import {
+    DEFAULT_DEDUP_WINDOW_MS,
+    DEFAULT_DEDUP_MAX_ENTRIES
+} from './constants.js';
+
+// ============================================================================
+// VALIDATION HELPER
+// ============================================================================
+
+/**
+ * Throws a classified INVALID_CONFIG error (ADR-018 fail-fast setup)
+ * when `value` is not a positive integer.
+ *
+ * @param {*} value - Candidate value
+ * @param {string} name - Field name for the error message
+ */
+const assertPositiveInteger = function ( value, name ) {
+    if ( typeof value !== 'number' || value < 1 || !Number.isInteger( value ) ) {
+        const err = new Error( `WinkComposer/mqtt-source: ${name} must be a positive integer` );
+        err.code = 'INVALID_CONFIG';
+        throw err;
+    }
+};
+
+// ============================================================================
+// DEDUP CACHE FACTORY
+// ============================================================================
+
+/**
+ * Create a time-bounded, count-capped dedup cache.
+ *
+ * All hot-path operations are O(1) (expiry amortized): Set lookup,
+ * ring-slot insertion, head eviction.
+ *
+ * @param {Object} [options={}] - Cache options
+ * @param {number} [options.windowMs=120000] - Time bound: an entry
+ *   expires once its age reaches this many milliseconds
+ * @param {number} [options.maxEntries=65536] - Count cap: the memory
+ *   guarantee; the oldest entry is evicted when the ring is full
+ * @param {function} [options.nowFn=Date.now] - Clock source. Injection
+ *   point for deterministic tests; production uses the default
+ * @returns {Object} Cache instance with isDuplicate, size, clear, has
+ */
+const createDedupCache = function ( options = {} ) {
+    if ( typeof options !== 'object' || options === null ) {
+        const err = new Error( 'WinkComposer/mqtt-source: dedup options must be an object' );
+        err.code = 'INVALID_CONFIG';
+        throw err;
+    }
+
+    const {
+        windowMs = DEFAULT_DEDUP_WINDOW_MS,
+        maxEntries = DEFAULT_DEDUP_MAX_ENTRIES,
+        nowFn = Date.now
+    } = options;
+
+    assertPositiveInteger( windowMs, 'windowMs' );
+    assertPositiveInteger( maxEntries, 'maxEntries' );
+    if ( typeof nowFn !== 'function' ) {
+        const err = new Error( 'WinkComposer/mqtt-source: nowFn must be a function' );
+        err.code = 'INVALID_CONFIG';
+        throw err;
+    }
+
+    // Pre-allocated structures (zero allocation on the hot path).
+    const seen = new Set();                           // O(1) duplicate lookup
+    const ids = new Array( maxEntries );              // Ring: dedup ids in arrival order
+    const stamps = new Float64Array( maxEntries );    // Ring: arrival timestamps
+    let head = 0;                                     // Oldest live entry
+    let tail = 0;                                     // Next insert slot
+    let count = 0;                                    // Live entries
+
+    /**
+     * Evict the entry at the ring head. The slot is cleared so the id
+     * string is not retained after eviction.
+     */
+    const evictHead = function () {
+        seen.delete( ids[ head ] );
+        ids[ head ] = undefined;
+        head = ( head + 1 ) % maxEntries;
+        count -= 1;
+    };
+
+    /**
+     * Check whether dedupId was seen within the window. New ids are
+     * cached; expired and cap-evicted ids count as new again.
+     *
+     * @param {string|null|undefined} dedupId - Deduplication identifier
+     * @returns {boolean} true if duplicate (skip), false if new (process)
+     */
+    const isDuplicate = function ( dedupId ) {
+        // No dedup id → bypass: the message is processed verbatim and
+        // the cache is not touched (opt-in by construction, ADR-022).
+        if ( dedupId === null || dedupId === undefined ) {
+            return false;
+        }
+
+        const now = nowFn();
+
+        // Sweep expired entries from the head. Bounded by the live
+        // count so the loop provably terminates; each entry is evicted
+        // at most once, so the sweep is amortized O(1) per call.
+        const live = count;
+        for ( let i = 0; i < live; i += 1 ) {
+            if ( ( now - stamps[ head ] ) < windowMs ) {
+                break;
+            }
+            evictHead();
+        }
+
+        if ( seen.has( dedupId ) ) {
+            return true;
+        }
+
+        // Cap bound: a full ring evicts its oldest to admit the newest.
+        if ( count === maxEntries ) {
+            evictHead();
+        }
+
+        ids[ tail ] = dedupId;
+        stamps[ tail ] = now;
+        seen.add( dedupId );
+        tail = ( tail + 1 ) % maxEntries;
+        count += 1;
+
+        return false;
+    };
+
+    /**
+     * Get current number of cached entries.
+     *
+     * @returns {number} Number of live entries
+     */
+    const size = function () {
+        return count;
+    };
+
+    /**
+     * Clear all cached entries. Used for testing or reset.
+     */
+    const clear = function () {
+        seen.clear();
+        ids.fill( undefined );
+        stamps.fill( 0 );
+        head = 0;
+        tail = 0;
+        count = 0;
+    };
+
+    /**
+     * Check if dedupId is in cache without modifying it. Test helper —
+     * does not add, evict, or sweep.
+     *
+     * @param {string} dedupId - Deduplication identifier
+     * @returns {boolean} true if in cache, false otherwise
+     */
+    const has = function ( dedupId ) {
+        return seen.has( dedupId );
+    };
+
+    return {
+        isDuplicate,
+        size,
+        clear,
+        has
+    };
+};
+
+// ============================================================================
+// EXPORTS
+// ============================================================================
+
+export { createDedupCache };

package/src/core/source-manager/mqtt/index.js

@@ -0,0 +1,260 @@
+// core/source-manager/mqtt/index.js
+
+/**
+ * @fileoverview MQTT source adapter for WinkComposer aggregator.
+ *
+ * Subscribes to edge composer MQTT outputs with deduplication for QoS 1.
+ * Extracts winkDedupId from MQTT v5 user properties to filter retransmissions.
+ *
+ * Adapter contract (ADR-018):
+ * - Module-level exports: `id`, `configSchema`, `start`,
+ *   `durabilityClass`, and the default aggregate referencing the same
+ *   constants.
+ * - `start( config )` returns a `stopFn({ timeout })`.
+ * - Lifecycle and error contract details: see `client.js` file header.
+ *
+ * Durability (ADR-018): `'broker-queue'` — the source connects with a
+ * persistent broker session (`cleanStart: false`), so while it is
+ * disconnected the broker holds every QoS-1 message published to its
+ * subscribed topics and replays them on reconnect. What the source has
+ * not yet acknowledged is not lost; the broker owns the queue.
+ *
+ * `configSchema` declares the user-supplied config fields. The flow
+ * runtime calls `validateWithSchema( mqtt.configSchema, sourceConfig )`
+ * at DSL time (in `flow.source()`), so typos and bad types are caught
+ * before the flow ever runs. Per ADR-018 the schema is authoritative
+ * for the fields it covers; `client.js` does not re-enforce them. The
+ * schema carries `_propertyNames`, so an unknown key (e.g. the typo
+ * `brokerURL`) is rejected at definition time.
+ *
+ * Runtime-injected callbacks (`onMessage`, `onShutdown`) are not part
+ * of `configSchema` — the wiring layer adds them to the config at start
+ * time, after DSL validation. A user-supplied `onMessage` would be
+ * silently overwritten; the unknown-key rejection turns that mistake
+ * into a fail-fast error instead.
+ *
+ *   ASSUMPTIONS
+ *   -----------
+ *   1. Inside a flow, the runtime injects `onMessage` / `onShutdown`
+ *      and wraps `onStatus`; a direct caller of `start()` supplies
+ *      `onMessage` itself and owns calling the returned stop function.
+ *   2. Config passed through the DSL was validated by this schema;
+ *      `client.js` re-checks only the three fields a direct caller
+ *      could omit (brokerUrl, topics, onMessage).
+ *
+ *   LIMITATIONS
+ *   -----------
+ *   1. This is an infinite source: it never emits `phase: 'complete'`,
+ *      so a flow's `whenComplete()` resolves only via `shutdown()`.
+ *   2. Duplicate filtering is opt-in per message: only messages that
+ *      carry `winkDedupId` are protected. Watch the `dedupBypassed`
+ *      counter to spot publishers that are not stamping ids.
+ *
+ * Usage in flow:
+ *   import mqtt from './core/source-manager/mqtt/index.js';
+ *
+ *   flow( 'aggregator' )
+ *       .source( mqtt, {
+ *           brokerUrl: 'mqtt://broker:1883',
+ *           topics: [ 'edge/+/enriched' ],
+ *           codec: msgpackCodec,
+ *           dedupWindowMs: 120000
+ *       } )
+ *
+ * @see src/core/emitter-manager/mqtt/index.js - Emitter counterpart
+ */
+
+import { validators } from '../../utils/validate/index.js';
+import { createMQTTSourceClient } from './client.js';
+
+// ============================================================================
+// SOURCE ADAPTER
+// ============================================================================
+
+/**
+ * Source adapter identifier.
+ * Used by the flow runtime to reference this adapter.
+ *
+ * @type {string}
+ */
+const id = 'mqtt';
+
+/**
+ * Crash-survival class per ADR-018. For a source the value describes
+ * the INPUT it can recover after a disconnect: the persistent broker
+ * session holds subscribed QoS-1 messages across the gap and replays
+ * them on reconnect.
+ *
+ * @type {string}
+ */
+const durabilityClass = 'broker-queue';
+
+/**
+ * Validates that a value is a non-empty topic string or a non-empty
+ * array of non-empty topic strings.
+ *
+ * @param {*} value
+ * @returns {boolean}
+ */
+const isTopicOrTopicArray = function ( value ) {
+    if ( typeof value === 'string' && value.length > 0 ) return true;
+    if ( Array.isArray( value ) && value.length > 0 ) {
+        return value.every( ( t ) => typeof t === 'string' && t.length > 0 );
+    }
+    return false;
+};
+
+/**
+ * Schema for the MQTT source's user-supplied config. Consumed by the
+ * flow runtime (`flow.source()` → `validateWithSchema`) at DSL time.
+ *
+ * Required:
+ * - `brokerUrl` — MQTT broker URL (e.g., 'mqtt://broker.local:1883').
+ * - `topics` — topic string or non-empty array of topic strings
+ *   (wildcards supported).
+ *
+ * Optional:
+ * - `codec` — payload decoder with an `unpack( buffer )` function.
+ *   Default: JSON.parse of the payload.
+ * - `transform` — function( msg ) → msg, applied before `onMessage`;
+ *   returning null/undefined drops the message (counted in `skipped`).
+ *   A throw skips that one message with a per-record CALLBACK_FAILED
+ *   report and the stream continues. Uniform with the CSV source.
+ * - `dedupWindowMs` — dedup time bound: a duplicate arriving within
+ *   this many milliseconds of its original is dropped. Default 120000
+ *   (ADR-022).
+ * - `dedupMaxEntries` — dedup count cap: the memory guarantee (~8 MB
+ *   worst case at the default 65536). At high rates the effective
+ *   window is maxEntries ÷ rate — see ADR-022.
+ * - `clientId` — MQTT client ID. Auto-generated when omitted.
+ * - `cleanStart` — override the persistent-session default (false).
+ * - `onStatus` — status callback. Receives the structured ADR-018
+ *   status payload on every transition and per decode failure — the full
+ *   vocabulary is documented in `client.js`.
+ * - `onMetrics` — counter-snapshot callback, ~1 Hz plus on every
+ *   health transition (delivered, skipped, decodeErrors, reconnects,
+ *   dedupHits, dedupMisses, dedupBypassed, dedupCacheSize).
+ * - `expectedQuietPeriodMs` — opt-in quiet rule: health goes yellow
+ *   when no packet arrives for longer than this many milliseconds.
+ *   Off when omitted (silence is normal for many sources).
+ * - `mqttConnectFn` — MQTT connect function injection point (tests,
+ *   benchmarks).
+ *
+ * `_propertyNames` lists every accepted key — unknown keys throw at
+ * DSL time (the validator's only unknown-key mechanism).
+ *
+ * @type {Object}
+ */
+const configSchema = {
+    _propertyNames: [
+        'brokerUrl',
+        'topics',
+        'codec',
+        'transform',
+        'dedupWindowMs',
+        'dedupMaxEntries',
+        'clientId',
+        'cleanStart',
+        'onStatus',
+        'onMetrics',
+        'expectedQuietPeriodMs',
+        'mqttConnectFn'
+    ],
+    brokerUrl: {
+        type: 'string',
+        required: true,
+        validator: validators.nonEmptyString,
+        error: 'brokerUrl must be a non-empty string (e.g., mqtt://broker.local:1883)'
+    },
+    topics: {
+        required: true,
+        validator: isTopicOrTopicArray,
+        error: 'topics must be a non-empty string or a non-empty array of non-empty strings'
+    },
+    codec: {
+        type: 'object',
+        required: false,
+        validator: ( v ) => v !== null && typeof v.unpack === 'function',
+        error: 'codec must be an object with an unpack() function'
+    },
+    transform: {
+        type: 'function',
+        required: false,
+        error: 'transform must be a function( msg ) returning a msg (or null/undefined to drop)'
+    },
+    dedupWindowMs: {
+        type: 'number',
+        required: false,
+        validator: validators.positiveInteger,
+        error: 'dedupWindowMs must be a positive integer (milliseconds; default 120000)'
+    },
+    dedupMaxEntries: {
+        type: 'number',
+        required: false,
+        validator: validators.positiveInteger,
+        error: 'dedupMaxEntries must be a positive integer (default 65536)'
+    },
+    clientId: {
+        type: 'string',
+        required: false,
+        validator: validators.noSpaces,
+        error: 'clientId must be a string without spaces'
+    },
+    cleanStart: {
+        type: 'boolean',
+        required: false,
+        error: 'cleanStart must be a boolean'
+    },
+    onStatus: {
+        type: 'function',
+        required: false,
+        error: 'onStatus must be a function'
+    },
+    onMetrics: {
+        type: 'function',
+        required: false,
+        error: 'onMetrics must be a function'
+    },
+    expectedQuietPeriodMs: {
+        type: 'number',
+        required: false,
+        validator: validators.positiveInteger,
+        error: 'expectedQuietPeriodMs must be a positive integer (milliseconds)'
+    },
+    mqttConnectFn: {
+        type: 'function',
+        required: false,
+        error: 'mqttConnectFn must be a function (test/benchmark injection point)'
+    }
+};
+
+/**
+ * Start the MQTT source.
+ *
+ * Called by the flow runtime (flow/run.js) with config from flow DSL.
+ * Returns a stop function for graceful shutdown.
+ *
+ * @param {Object} config - Source configuration
+ * @param {string} config.brokerUrl - MQTT broker URL
+ * @param {string|string[]} config.topics - Topic(s) to subscribe to
+ * @param {function} config.onMessage - Message handler (injected by the flow runtime)
+ * @param {Object} [config.codec] - Codec for payload decoding
+ * @param {function} [config.transform] - Optional message transform
+ * @param {number} [config.dedupWindowMs=120000] - Dedup time bound (ADR-022)
+ * @param {number} [config.dedupMaxEntries=65536] - Dedup count cap (ADR-022)
+ * @param {string} [config.clientId] - MQTT client ID
+ * @param {function} [config.onStatus] - Structured status callback (ADR-018)
+ * @param {function} [config.onMetrics] - Counter-snapshot callback (~1 Hz + transitions)
+ * @param {number} [config.expectedQuietPeriodMs] - Opt-in quiet rule threshold
+ * @returns {function} Stop function that returns a Promise
+ */
+const start = function ( config ) {
+    return createMQTTSourceClient( config );
+};
+
+// ============================================================================
+// EXPORTS
+// ============================================================================
+
+export { id, configSchema, durabilityClass, start };
+export default { id, configSchema, durabilityClass, start };