@yarkivaev/scada 2.3.115 → 2.4.0

package/src/infrastructure/ingest/modbus/silentStreams.js

@@ -1,5 +1,20 @@
-import { clock } from '@yarkivaev/source-to-sink';
+import { clock, intervalSource } from '@yarkivaev/source-to-sink';
 
+function track(state, pending) {
+    if (!pending || typeof pending.then !== 'function') { return; }
+    const task = pending.catch((error) => {
+        state.failure = error;
+        try { state.onError?.(error); } catch {
+            // Reporting must not turn an observed source failure into an unhandled rejection.
+        }
+    })
+        .finally(() => { state.pending.delete(task); });
+    state.pending.add(task);
+}
+async function drained(state) {
+    await Promise.all(state.pending);
+    if (state.failure) { throw state.failure; }
+}
 /**
  * Whether a stream is past the silence budget.
  *
@@ -27,8 +42,12 @@ function stopOne(state, name) {
     if (!state.live.has(name)) {
         return;
     }
-    state.held.get(name).stop();
-    state.live.delete(name);
+    if (state.closing.has(name)) { return; }
+    const pending = state.held.get(name).stop();
+    if (pending && typeof pending.then === 'function') {
+        state.closing.add(name);
+        track(state, pending.finally(() => { state.live.delete(name); state.closing.delete(name); }));
+    } else { state.live.delete(name); }
 }
 
 /**
@@ -47,7 +66,7 @@ function startOne(state, source) {
         poll = source.open(state.collector, state.clk, state.intervalSec);
         state.held.set(name, poll);
     }
-    poll.start();
+    track(state, poll.start());
     state.live.add(name);
 }
 
@@ -83,15 +102,16 @@ function gateState(config) {
         intervalSec: interval,
         sources,
         clk: config.clock || clock(),
-        delay: config.delay || ((fn, ms) => {
-            return setInterval(fn, ms);
-        }),
-        clear: config.clear || clearInterval,
+        delay: config.delay || setTimeout,
+        clear: config.clear || clearTimeout,
         lastSeen: new Map(),
         held: new Map(),
         live: new Set(),
+        closing: new Set(),
+        pending: new Set(),
+        failure: null,
+        onError: config.onError,
         collector: undefined,
-        timer: undefined,
         running: false
     };
 }
@@ -119,6 +139,9 @@ export default function silentStreams(config) {
     function runPulse() {
         pulse(state);
     }
+    const ticker = intervalSource(state.intervalSec * 1000, { accept: runPulse }, {
+        schedule: state.delay, clear: state.clear
+    });
     return {
         seen(name, at = state.clk.millis()) {
             state.lastSeen.set(name, at);
@@ -129,24 +152,24 @@ export default function silentStreams(config) {
         bind(sink) {
             state.collector = {
                 accept(record) {
-                    sink.write([record]);
+                    return sink.write([record]);
                 }
             };
         },
         start() {
+            if (state.running) { return drained(state); }
             state.running = true;
             runPulse();
-            state.timer = state.delay(runPulse, state.intervalSec * 1000);
+            track(state, ticker.start());
+            return drained(state);
         },
-        stop() {
+        async stop() {
             state.running = false;
-            if (state.timer !== undefined) {
-                state.clear(state.timer);
-                state.timer = undefined;
-            }
+            track(state, ticker.stop());
             for (const name of [...state.live]) {
                 stopOne(state, name);
             }
+            await drained(state);
             state.held.clear();
         },
         pulse: runPulse

package/src/infrastructure/ingest/mqtt/metricsTransformer.js

@@ -30,25 +30,24 @@ export default function metricsCodec(collector, log = console) {
                 parsed = JSON.parse(raw.payload);
             } catch {
                 log.warn(`Cannot resolve payload on ${raw.topic}: not valid JSON`);
-                return;
+                return undefined;
             }
             if (typeof parsed === 'number') {
-                collector.accept({
+                return collector.accept({
                     topic: raw.topic,
                     ts: Date.now(),
                     value: parsed
                 });
-                return;
             }
             if (typeof parsed === 'object' && parsed !== null && typeof parsed.value === 'number') {
-                collector.accept({
+                return collector.accept({
                     topic: raw.topic,
                     ts: parsed.ts || Date.now(),
                     value: parsed.value
                 });
-                return;
             }
             log.warn(`Cannot resolve payload on ${raw.topic}: no numeric value found`);
+            return undefined;
         }
     };
 }

package/src/infrastructure/ingest/mqtt/modbusMqtt.js

@@ -1,12 +1,6 @@
-import {
-    batch,
-    circuit,
-    clock,
-    modbusRtuBusSource,
-    modbusRtuSource,
-    modbusSource,
-    mqttSink
-} from '@yarkivaev/source-to-sink';
+import { clock } from '@yarkivaev/source-to-sink';
+import { modbusRtuBusSource, modbusRtuSource, modbusSource, mqttSink } from '../../streams/index.js';
+import ingestBatch from '../ingestBatch.js';
 import { parseModbusDeviceSpec } from './modbusDeviceSpec.js';
 
 /**
@@ -181,24 +175,28 @@ function buildSources(devices, config, collector, clk) {
 export default function modbusMqtt(mqtt, devices, config) {
     assertModbusMqttArgs(mqtt, devices, config);
     const clk = clock();
-    const breaker = circuit(config.threshold, config.timeout, clk);
     const sink = mqttSink(mqtt, {
         clientId: config.clientId || 'scada-modbus',
         qos: 1
     });
-    const collector = batch(sink, 5, breaker);
+    const collector = ingestBatch(sink, { ...config, size: config.size ?? 5 });
     const sources = buildSources(devices, config, collector, clk);
     return {
-        start() {
-            sink.start();
-            for (const source of sources) {
-                source.start();
+        async start() {
+            await sink.start();
+            try {
+                await Promise.all(sources.map((source) => { return source.start(); }));
+            } catch (error) {
+                await Promise.allSettled(sources.map((source) => { return source.stop(); }));
+                await sink.stop();
+                throw error;
             }
         },
-        stop() {
-            for (const source of sources) {
-                source.stop();
-            }
+        async stop() {
+            try {
+                await Promise.all(sources.map((source) => { return source.stop(); }));
+                await collector.stop();
+            } finally { await sink.stop(); }
         }
     };
 }

package/src/infrastructure/ingest/mqtt/modbusPublish.js

@@ -260,6 +260,10 @@ export default function modbusPublish(mqtt, config) {
         return [row.name, plans[index]];
     }));
     return modbusMqtt(mqtt, specs, {
+        budget: config.budget,
+        maxRecords: config.maxRecords,
+        maxBytes: config.maxBytes,
+        writeTimeoutMs: config.writeTimeoutMs,
         address: frame.address,
         clientId: config.clientId || 'scada-modbus',
         count: frame.count,

package/src/infrastructure/ingest/mqtt/mqttMetrics.js

@@ -1,21 +1,16 @@
-import {
-    batch,
-    circuit,
-    clock,
-    mqttSource,
-    timedBatch
-} from '@yarkivaev/source-to-sink';
+import { mqttSource } from '../../streams/index.js';
+import ingestBatch from '../ingestBatch.js';
 import metricsCodec from './metricsTransformer.js';
 
 /**
  * Pipeline for streaming MQTT sensor data to a storage sink.
  *
- * Subscribes to MQTT topic and batches messages before writing
- * to the provided sink. Uses circuit breaker for failure isolation
- * and time-based flushing for low-volume periods.
+ * Subscribes to MQTT and writes each message before acknowledging it.
+ * MQTT.js serializes acknowledgement, so acceptance cannot wait for a larger batch.
+ * Bounds queued records and bytes and enforces a durable-write deadline.
  *
  * @example
- *   import { clickhouseSink } from '@yarkivaev/source-to-sink';
+ *   import { clickhouseSink } from '../../streams/index.js';
  *   const sink = clickhouseSink('http://localhost:8123', 'scada.metrics');
  *   const pipeline = mqttMetrics(
  *     'mqtt://localhost:1883',
@@ -31,10 +26,10 @@ import metricsCodec from './metricsTransformer.js';
  * @param {object} sink - Sink with write(records) method
  * @param {string} topic - MQTT topic pattern to subscribe
  * @param {object} config - Pipeline configuration
- * @param {number} config.size - Batch size before flush
+ * @param {number} config.size - Legacy option; serial durable acceptance writes one record at a time
  * @param {number} config.interval - Seconds before time-based flush
- * @param {number} config.threshold - Circuit breaker failure threshold
- * @param {number} config.timeout - Circuit breaker timeout in seconds
+ * @param {number} config.threshold - Legacy option (retained for configuration compatibility)
+ * @param {number} config.timeout - Legacy option (use writeTimeoutMs for write deadlines)
  * @param {string} [config.clientId] - MQTT client ID for persistent sessions
  * @param {number} [config.sessionExpiryInterval] - Session expiry in seconds (default 3600)
  * @returns {object} Pipeline with start() and stop() methods
@@ -52,11 +47,12 @@ export default function mqttMetrics(mqtt, sink, topic, config) {
     if (!config || typeof config !== 'object') {
         throw new Error('Config must be an object');
     }
-    const clk = clock();
-    const breaker = circuit(config.threshold, config.timeout, clk);
-    const collector = timedBatch(batch(sink, config.size, breaker), config.interval);
+    const collector = ingestBatch(sink, { ...config, size: 1 });
     const transformer = metricsCodec(collector);
     const source = mqttSource(mqtt, topic, transformer, {
+        budget: config.budget,
+        prefetch: config.prefetch,
+        maxPacketBytes: config.maxBytes,
         clientId: config.clientId,
         sessionExpiryInterval: config.sessionExpiryInterval
     });
@@ -65,14 +61,14 @@ export default function mqttMetrics(mqtt, sink, topic, config) {
          * Starts the pipeline.
          */
         start() {
-            source.start();
+            return source.start();
         },
         /**
          * Stops the pipeline.
          */
-        stop() {
-            source.stop();
-            collector.stop();
+        async stop() {
+            await source.stop();
+            await collector.stop();
         }
     };
 }

package/src/infrastructure/ingest/mqtt/registerPublish.js

@@ -70,15 +70,15 @@ export default function registerPublish(plans, origin, collector) {
          * @param {Array<number>} registers - uint16 window
          */
         accept(registers) {
-            rows.forEach((item) => {
-                collector.accept({
+            return Promise.all(rows.map((item) => {
+                return collector.accept({
                     payload: JSON.stringify({
                         ts: Date.now(),
                         value: decodePoint(item.point, origin, registers)
                     }),
                     topic: item.topic
                 });
-            });
+            }));
         }
     };
 }

package/src/infrastructure/ingest/pipelines/alertPipeline.js

@@ -1,4 +1,5 @@
-import { stompSource } from '@yarkivaev/source-to-sink';
+import stompOptions from './stompOptions.js';
+import { stompSource } from '../../streams/index.js';
 import alertCodec from '../codecs/alertCodec.js';
 import alertSink from '../sinks/alertSink.js';
 
@@ -29,20 +30,20 @@ export default function alertPipeline(stomp, pool, config) {
     const sink = alertSink(pool);
     const codec = alertCodec(sink, config.translations || {});
     const source = stompSource(stomp, '/exchange/scada.alerts', codec,
-        { login: config.login, passcode: config.passcode, host: config.host });
+        stompOptions(config));
     return {
         /**
          * Starts the pipeline.
          */
         start() {
-            source.start();
+            return source.start();
         },
         /**
          * Stops the pipeline.
          */
-        stop() {
-            source.stop();
-            sink.stop();
+        async stop() {
+            await source.stop();
+            await sink.stop();
         }
     };
 }

package/src/infrastructure/ingest/pipelines/decisionPipeline.js

@@ -1,4 +1,5 @@
-import { stompSource } from '@yarkivaev/source-to-sink';
+import stompOptions from './stompOptions.js';
+import { stompSource } from '../../streams/index.js';
 import userDecisionCodec from '../codecs/userDecisionCodec.js';
 import userDecisionSink from '../sinks/userDecisionSink.js';
 
@@ -26,20 +27,20 @@ export default function decisionPipeline(stomp, pool, config) {
     const sink = userDecisionSink(pool);
     const codec = userDecisionCodec(sink);
     const source = stompSource(stomp, '/exchange/scada.user_decisions', codec,
-        { login: config.login, passcode: config.passcode, host: config.host });
+        stompOptions(config));
     return {
         /**
          * Starts consuming user decisions from RabbitMQ.
          */
         start() {
-            source.start();
+            return source.start();
         },
         /**
          * Stops consuming and releases resources.
          */
-        stop() {
-            source.stop();
-            pool.end();
+        async stop() {
+            await source.stop();
+            await pool.end();
         }
     };
 }

package/src/infrastructure/ingest/pipelines/segmentPipeline.js

@@ -1,7 +1,5 @@
-import {
-    postgresSink,
-    stompSource
-} from '@yarkivaev/source-to-sink';
+import stompOptions from './stompOptions.js';
+import { postgresSink, stompSource } from '../../streams/index.js';
 import segmentDispatch from '../../../domain/segment/dispatch.js';
 import segmentCoalesce from '../../../domain/segment/segmentCoalesce.js';
 import silenceBudget from '../../../domain/segment/silenceBudget.js';
@@ -64,14 +62,7 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
         interval: config.interval
     });
     const destination = config.segmentsDestination || segmentsIngestDestination;
-    const source = stompSource(stomp, destination, coalesce, {
-        login: config.login,
-        passcode: config.passcode,
-        host: config.host,
-        serial: false,
-        manualAck: true,
-        prefetch: config.prefetch
-    });
+    const source = stompSource(stomp, destination, coalesce, stompOptions(config, { serial: false, manualAck: true }));
     const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
         intervalMs: (config.poll || 5) * 1000
     });
@@ -82,13 +73,13 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
             void repair.catch((error) => {
                 processingErrorLog('abut_segments', error, { budgetSeconds: budget });
             });
-            source.start();
+            await source.start();
             await silence.start();
         },
-        stop() {
-            silence.stop();
-            source.stop();
-            pool.end();
+        async stop() {
+            await silence.stop();
+            await source.stop();
+            await pool.end();
         }
     };
 }

package/src/infrastructure/ingest/pipelines/stompOptions.js

@@ -0,0 +1,7 @@
+/** Preserve driver defaults when an optional pipeline setting was omitted. */
+export default function stompOptions(config, overrides = {}) {
+    const options = { login: config.login, passcode: config.passcode, host: config.host,
+        prefetch: config.prefetch, maxMessageBytes: config.maxMessageBytes,
+        maxPendingBytes: config.maxPendingBytes, budget: config.budget, ...overrides };
+    return Object.fromEntries(Object.entries(options).filter(([, value]) => { return value !== undefined; }));
+}

package/src/infrastructure/ingest/silentOpenWatch.js

@@ -1,3 +1,5 @@
+import { intervalSource } from '@yarkivaev/source-to-sink';
+
 /**
  * Periodic wall-clock sweep that closes stale open segments without waiting for STOMP.
  *
@@ -18,46 +20,7 @@ export default function silentOpenWatch(closer, budget, options) {
         throw new Error(`Budget must be a positive number: ${budget}`);
     }
     const intervalMs = options && options.intervalMs ? options.intervalMs : 5000;
-    const gate = { busy: false };
-    let timer;
-    let active = false;
-    function claim() {
-        const free = !gate.busy;
-        gate.busy = true;
-        return free;
-    }
-    function release() {
-        gate.busy = false;
-    }
-    async function tick() {
-        if (!claim()) {
-            return;
-        }
-        try {
-            await closer.close(budget);
-        } catch {
-            /* next tick retries */
-        }
-        release();
-    }
-    return {
-        async start() {
-            if (active) {
-                return;
-            }
-            active = true;
-            await tick();
-            timer = setInterval(() => {
-                void tick();
-            }, intervalMs);
-        },
-        stop() {
-            if (!active) {
-                return;
-            }
-            active = false;
-            clearInterval(timer);
-            timer = undefined;
-        }
-    };
+    return intervalSource(intervalMs, {
+        accept() { return closer.close(budget); }
+    }, { immediate: true });
 }

package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js

@@ -1,5 +1,5 @@
-import { batch, circuit, clock, timedBatch } from '@yarkivaev/source-to-sink';
-import durableConsume from '../durableConsume.js';
+import { amqpSource } from '../../streams/index.js';
+import ingestBatch from '../ingestBatch.js';
 import deliverToMqttRecord from './deliverToMqttRecord.js';
 import metricsCodec from '../mqtt/metricsTransformer.js';
 import streamNameFromTopic from './streamNameFromTopic.js';
@@ -20,7 +20,7 @@ export function acceptTelemetryDeliver(codec, fields, content, onSeen) {
             onSeen(name);
         }
     }
-    codec.accept({ topic: record.topic, payload: record.payload.toString() });
+    return codec.accept({ topic: record.topic, payload: record.payload.toString() });
 }
 
 /**
@@ -34,10 +34,14 @@ export function acceptTelemetryDeliver(codec, fields, content, onSeen) {
 export function telemetryConsumer(codec, channel, onSeen) {
     return (msg) => {
         if (!msg) {
-            return;
+            return undefined;
+        }
+        const accepted = acceptTelemetryDeliver(codec, msg.fields, msg.content, onSeen);
+        if (accepted && typeof accepted.then === 'function') {
+            return accepted.then(() => { channel.ack(msg); });
         }
-        acceptTelemetryDeliver(codec, msg.fields, msg.content, onSeen);
         channel.ack(msg);
+        return undefined;
     };
 }
 
@@ -50,38 +54,22 @@ export function telemetryConsumer(codec, channel, onSeen) {
  * @param {object} [options] - batch and prefetch options
  * @returns {object} ingest with start and stop
  */
-function subscribeMetrics(queue, prefetch, codec, onSeen) {
-    return async (channel) => {
-        await channel.assertQueue(queue, { durable: true });
-        channel.prefetch(prefetch);
-        const onMessage = telemetryConsumer(codec, channel, onSeen);
-        const tag = await channel.consume(queue, onMessage, { noAck: false });
-        return tag.consumerTag;
-    };
-}
-
 export default function amqpMetricsIngest(amqpUrl, queue, sink, options = {}) {
     const prefetch = options.prefetch || 32;
-    const size = options.size || 100;
-    const interval = options.interval || 5;
-    const threshold = options.threshold || 5;
-    const timeout = options.timeout || 60;
-    const clk = clock();
-    const breaker = circuit(threshold, timeout, clk);
-    const collector = timedBatch(batch(sink, size, breaker), interval);
+    const collector = ingestBatch(sink, options);
     const codec = metricsCodec(collector);
-    const session = durableConsume(
-        amqpUrl,
-        subscribeMetrics(queue, prefetch, codec, options.onSeen),
-        options
-    );
+    const session = amqpSource(amqpUrl, queue, {
+        accept(delivery) {
+            return acceptTelemetryDeliver(codec, delivery.fields, delivery.content, options.onSeen);
+        }
+    }, { ...options, prefetch });
     return {
         start() {
             return session.start();
         },
         async stop() {
             await session.stop();
-            collector.stop();
+            await collector.stop();
         }
     };
 }

package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js

@@ -1,4 +1,4 @@
-import durableConsume from '../durableConsume.js';
+import { amqpSource } from '../../streams/index.js';
 import deliverToMqttRecord from './deliverToMqttRecord.js';
 
 /**
@@ -11,11 +11,15 @@ import deliverToMqttRecord from './deliverToMqttRecord.js';
 export function relayConsumer(sink, channel) {
     return (msg) => {
         if (!msg) {
-            return;
+            return undefined;
         }
         const record = deliverToMqttRecord(msg.fields, msg.content);
-        sink.write([record]);
+        const written = sink.write([record]);
+        if (written && typeof written.then === 'function') {
+            return written.then(() => { channel.ack(msg); });
+        }
         channel.ack(msg);
+        return undefined;
     };
 }
 
@@ -23,7 +27,7 @@ export function relayConsumer(sink, channel) {
  * AMQP queue consumer that republishes messages to MQTT using routing key as topic.
  *
  * @example
- *   import { mqttSink } from '@yarkivaev/source-to-sink';
+ *   import { mqttSink } from '../../streams/index.js';
  *   const sink = mqttSink('mqtt://localhost:1883', { qos: 1 });
  *   const relay = amqpMqttRelay('amqp://localhost', 'scada.telemetry.ingest', sink);
  *   relay.start();
@@ -34,35 +38,26 @@ export function relayConsumer(sink, channel) {
  * @param {object} [options] - prefetch count
  * @returns {object} Relay with start and stop
  */
-function subscribeRelay(queue, prefetch, sink) {
-    return async (channel) => {
-        await channel.assertQueue(queue, { durable: true });
-        channel.prefetch(prefetch);
-        const onMessage = relayConsumer(sink, channel);
-        const tag = await channel.consume(queue, onMessage, { noAck: false });
-        return tag.consumerTag;
-    };
-}
-
 export default function amqpMqttRelay(amqpUrl, queue, sink, options = {}) {
     const prefetch = options.prefetch || 32;
     let opened = false;
-    const session = durableConsume(
-        amqpUrl,
-        subscribeRelay(queue, prefetch, sink),
-        options
-    );
+    const session = amqpSource(amqpUrl, queue, {
+        accept(delivery) {
+            return sink.write([deliverToMqttRecord(delivery.fields, delivery.content)]);
+        }
+    }, { ...options, prefetch });
     return {
-        start() {
+        async start() {
             if (!opened) {
+                await sink.start();
                 opened = true;
-                sink.start();
             }
             return session.start();
         },
         async stop() {
             await session.stop();
-            sink.stop();
+            await sink.stop();
+            opened = false;
         }
     };
 }

package/src/infrastructure/ingest/telemetry/foldingSink.js

@@ -12,9 +12,12 @@
  */
 export default function foldingSink(inner, fold, locate) {
     return {
-        async write(records) {
-            await inner.write(records);
+        stop() { return inner.stop?.(); },
+        async write(records, context = {}) {
+            context.signal?.throwIfAborted();
+            await inner.write(records, context);
             for (const record of records) {
+                context.signal?.throwIfAborted();
                 const sample = locate(record);
                 if (sample) {
                     // eslint-disable-next-line no-await-in-loop

package/src/infrastructure/messaging/stomp/segmentRetags.js

@@ -1,4 +1,4 @@
-import { stompSend } from '@yarkivaev/source-to-sink';
+import { stompSend } from '../../streams/index.js';
 
 const DEFAULT_DESTINATION = '/exchange/scada.segments';
 

package/src/infrastructure/messaging/stomp/userDecisions.js

@@ -1,4 +1,4 @@
-import { stompSend } from '@yarkivaev/source-to-sink';
+import { stompSend } from '../../streams/index.js';
 import userDecisionBody from './userDecisionBody.js';
 
 const DEFAULT_DESTINATION = '/exchange/scada.user_decisions';