@yarkivaev/scada 1.5.0 → 2.3.45

package/src/infrastructure/ingest/mqtt/modbusMqtt.js

@@ -0,0 +1,204 @@
+import {
+    batch,
+    circuit,
+    clock,
+    modbusRtuBusSource,
+    modbusRtuSource,
+    modbusSource,
+    mqttSink
+} from '@yarkivaev/source-to-sink';
+import { parseModbusDeviceSpec } from './modbusDeviceSpec.js';
+
+/**
+ * Builds a TCP Modbus polling source for one device.
+ *
+ * @param {object} device - Parsed TCP device descriptor
+ * @param {object} config - Pipeline config
+ * @param {object} collector - Batch collector
+ * @param {object} clk - Clock instance
+ * @returns {object} Modbus polling source
+ */
+function buildTcpSource(device, config, collector, clk) {
+    const transformer = config.transformerFactory(device.name, collector);
+    return modbusSource(
+        device.host,
+        device.port,
+        config.address,
+        config.count,
+        config.interval,
+        transformer,
+        clk
+    );
+}
+
+/**
+ * Serial bus key for grouping RTU devices that share one port.
+ *
+ * @param {object} device - Parsed RTU device descriptor
+ * @returns {string} Group key
+ */
+function rtuBusKey(device) {
+    const { baudRate, dataBits, stopBits, parity } = device.serial;
+    return `${device.path}|${baudRate}|${dataBits}|${stopBits}|${parity}`;
+}
+
+/**
+ * Groups RTU devices by shared serial port and line settings.
+ *
+ * @param {Array<object>} rtuDevices - Parsed RTU device descriptors
+ * @returns {Array<Array<object>>} Device groups
+ */
+function groupRtuDevices(rtuDevices) {
+    const groups = new Map();
+    for (const device of rtuDevices) {
+        const key = rtuBusKey(device);
+        const list = groups.get(key) || [];
+        list.push(device);
+        groups.set(key, list);
+    }
+    return [...groups.values()];
+}
+
+/**
+ * Builds a single-slave RTU polling source.
+ *
+ * @param {object} device - Parsed RTU device
+ * @param {object} config - Pipeline config
+ * @param {object} collector - Batch collector
+ * @param {object} clk - Clock instance
+ * @returns {object} Modbus RTU source
+ */
+function buildSingleRtuSource(device, config, collector, clk) {
+    return modbusRtuSource(
+        device.path,
+        device.serial,
+        config.address,
+        config.count,
+        config.interval,
+        config.transformerFactory(device.name, collector),
+        clk
+    );
+}
+
+/**
+ * Builds a multi-slave RTU bus source for one serial port.
+ *
+ * @param {Array<object>} devices - RTU devices on the same port
+ * @param {object} config - Pipeline config
+ * @param {object} collector - Batch collector
+ * @returns {object} Modbus RTU bus source
+ */
+function buildBusRtuSource(devices, config, collector) {
+    const slaves = devices.map((device) => {
+        return {
+            slaveId: device.serial.slaveId,
+            collector: config.transformerFactory(device.name, collector)
+        };
+    });
+    const first = devices[0];
+    return modbusRtuBusSource(first.path, first.serial, slaves, {
+        address: config.address,
+        count: config.count,
+        interval: config.interval
+    });
+}
+
+/**
+ * Builds RTU sources, merging slaves on the same serial port into one bus.
+ *
+ * @param {Array<object>} rtuDevices - Parsed RTU device descriptors
+ * @param {object} config - Pipeline config
+ * @param {object} collector - Batch collector
+ * @param {object} clk - Clock instance
+ * @returns {Array<object>} Modbus polling sources
+ */
+function buildRtuSources(rtuDevices, config, collector, clk) {
+    return groupRtuDevices(rtuDevices).map((devices) => {
+        if (devices.length === 1) {
+            return buildSingleRtuSource(devices[0], config, collector, clk);
+        }
+        return buildBusRtuSource(devices, config, collector);
+    });
+}
+
+/**
+ * Validates modbusMqtt constructor arguments.
+ *
+ * @param {string} mqtt - MQTT broker URL
+ * @param {string} devices - Device specs string
+ * @param {object} config - Pipeline config
+ */
+function assertModbusMqttArgs(mqtt, devices, config) {
+    if (typeof mqtt !== 'string' || mqtt.length === 0) {
+        throw new Error('MQTT URL must be a non-empty string');
+    }
+    if (typeof devices !== 'string' || devices.length === 0) {
+        throw new Error('Devices must be a non-empty string');
+    }
+    if (!config || typeof config !== 'object') {
+        throw new Error('Config must be an object');
+    }
+    if (typeof config.transformerFactory !== 'function') {
+        throw new Error('transformerFactory is required for modbusMqtt');
+    }
+}
+
+/**
+ * Parses device specs and builds TCP plus RTU polling sources.
+ *
+ * @param {string} devices - Comma-separated device specs
+ * @param {object} config - Pipeline config
+ * @param {object} collector - Batch collector
+ * @param {object} clk - Clock instance
+ * @returns {Array<object>} Modbus polling sources
+ */
+function buildSources(devices, config, collector, clk) {
+    const parsed = devices.split(',').map((spec) => {
+        return parseModbusDeviceSpec(spec.trim());
+    });
+    const tcp = parsed.filter((device) => {
+        return device.kind === 'tcp';
+    });
+    const rtu = parsed.filter((device) => {
+        return device.kind === 'rtu';
+    });
+    return [
+        ...tcp.map((device) => {
+            return buildTcpSource(device, config, collector, clk);
+        }),
+        ...buildRtuSources(rtu, config, collector, clk)
+    ];
+}
+
+/**
+ * Pipeline for polling Modbus devices and publishing to MQTT.
+ *
+ * @param {string} mqtt - MQTT broker URL
+ * @param {string} devices - Comma-separated specs: name:host:port or name:rtu:path:baud[:line][:slaveId]
+ * @param {object} config - interval, address, count, threshold, timeout, clientId, transformerFactory
+ * @returns {object} Pipeline with start() and stop() methods
+ */
+export default function modbusMqtt(mqtt, devices, config) {
+    assertModbusMqttArgs(mqtt, devices, config);
+    const clk = clock();
+    const breaker = circuit(config.threshold, config.timeout, clk);
+    const sink = mqttSink(mqtt, {
+        clientId: config.clientId || 'scada-modbus',
+        qos: 1
+    });
+    const collector = batch(sink, 5, breaker);
+    const sources = buildSources(devices, config, collector, clk);
+    return {
+        start() {
+            sink.start();
+            for (const source of sources) {
+                source.start();
+            }
+        },
+        stop() {
+            for (const source of sources) {
+                source.stop();
+            }
+        }
+    };
+}

package/src/infrastructure/ingest/mqtt/mqttMetrics.js

@@ -0,0 +1,78 @@
+import {
+    batch,
+    circuit,
+    clock,
+    mqttSource,
+    timedBatch
+} from '@yarkivaev/source-to-sink';
+import metricsCodec from './metricsTransformer.js';
+
+/**
+ * Pipeline for streaming MQTT sensor data to a storage sink.
+ *
+ * Subscribes to MQTT topic and batches messages before writing
+ * to the provided sink. Uses circuit breaker for failure isolation
+ * and time-based flushing for low-volume periods.
+ *
+ * @example
+ *   import { clickhouseSink } from '@yarkivaev/source-to-sink';
+ *   const sink = clickhouseSink('http://localhost:8123', 'scada.metrics');
+ *   const pipeline = mqttMetrics(
+ *     'mqtt://localhost:1883',
+ *     sink,
+ *     'sensors/#',
+ *     { size: 100, interval: 5, threshold: 5, timeout: 60 }
+ *   );
+ *   pipeline.start();
+ *   // ... later
+ *   pipeline.stop();
+ *
+ * @param {string} mqtt - MQTT broker URL
+ * @param {object} sink - Sink with write(records) method
+ * @param {string} topic - MQTT topic pattern to subscribe
+ * @param {object} config - Pipeline configuration
+ * @param {number} config.size - Batch size before flush
+ * @param {number} config.interval - Seconds before time-based flush
+ * @param {number} config.threshold - Circuit breaker failure threshold
+ * @param {number} config.timeout - Circuit breaker timeout in seconds
+ * @param {string} [config.clientId] - MQTT client ID for persistent sessions
+ * @param {number} [config.sessionExpiryInterval] - Session expiry in seconds (default 3600)
+ * @returns {object} Pipeline with start() and stop() methods
+ */
+export default function mqttMetrics(mqtt, sink, topic, config) {
+    if (typeof mqtt !== 'string' || mqtt.length === 0) {
+        throw new Error('MQTT URL must be a non-empty string');
+    }
+    if (!sink || typeof sink.write !== 'function') {
+        throw new Error('Sink must have a write(records) method');
+    }
+    if (typeof topic !== 'string' || topic.length === 0) {
+        throw new Error('Topic must be a non-empty string');
+    }
+    if (!config || typeof config !== 'object') {
+        throw new Error('Config must be an object');
+    }
+    const clk = clock();
+    const breaker = circuit(config.threshold, config.timeout, clk);
+    const collector = timedBatch(batch(sink, config.size, breaker), config.interval);
+    const transformer = metricsCodec(collector);
+    const source = mqttSource(mqtt, topic, transformer, {
+        clientId: config.clientId,
+        sessionExpiryInterval: config.sessionExpiryInterval
+    });
+    return {
+        /**
+         * Starts the pipeline.
+         */
+        start() {
+            source.start();
+        },
+        /**
+         * Stops the pipeline.
+         */
+        stop() {
+            source.stop();
+            collector.stop();
+        }
+    };
+}

package/src/infrastructure/ingest/parseRequestTimeoutMs.js

@@ -0,0 +1,20 @@
+const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
+
+/**
+ * Parses REQUEST_TIMEOUT_MS from environment into a positive millisecond value.
+ *
+ * @param {string|undefined} raw - env value
+ * @returns {number} timeout in milliseconds
+ */
+export function parseRequestTimeoutMs(raw) {
+    if (raw === undefined || raw === null || raw === '') {
+        return DEFAULT_REQUEST_TIMEOUT_MS;
+    }
+    const ms = Number.parseInt(String(raw), 10);
+    if (!Number.isFinite(ms) || ms <= 0) {
+        return DEFAULT_REQUEST_TIMEOUT_MS;
+    }
+    return ms;
+}
+
+export { DEFAULT_REQUEST_TIMEOUT_MS };

package/src/infrastructure/ingest/pipelines/alertPipeline.js

@@ -0,0 +1,48 @@
+import { stompSource } from '@yarkivaev/source-to-sink';
+import alertCodec from '../codecs/alertCodec.js';
+import alertSink from '../sinks/alertSink.js';
+
+/**
+ * Pipeline for streaming STOMP alert data to PostgreSQL alerts table.
+ *
+ * Subscribes to STOMP exchange and forwards alert messages directly
+ * to the alert sink without batching. Alerts are rare and require
+ * per-record INSERT/UPDATE branching, making batching unnecessary.
+ *
+ * Plant packages pass human-readable messages via config.translations
+ * (rule name → message). Without translations, the rule name is stored.
+ *
+ * @example
+ *   const pipeline = alertPipeline(
+ *     'stomp://rabbitmq:61613',
+ *     pool,
+ *     { login: 'guest', passcode: 'guest', host: '/', translations: { no_data: 'No data' } }
+ *   );
+ *   pipeline.start();
+ *
+ * @param {string} stomp - STOMP broker URL
+ * @param {object} pool - PostgreSQL pool
+ * @param {object} config - Pipeline configuration
+ * @returns {object} Pipeline with start() and stop() methods
+ */
+export default function alertPipeline(stomp, pool, config) {
+    const sink = alertSink(pool);
+    const codec = alertCodec(sink, config.translations || {});
+    const source = stompSource(stomp, '/exchange/scada.alerts', codec,
+        { login: config.login, passcode: config.passcode, host: config.host });
+    return {
+        /**
+         * Starts the pipeline.
+         */
+        start() {
+            source.start();
+        },
+        /**
+         * Stops the pipeline.
+         */
+        stop() {
+            source.stop();
+            sink.stop();
+        }
+    };
+}

package/src/infrastructure/ingest/pipelines/decisionPipeline.js

@@ -0,0 +1,45 @@
+import { stompSource } from '@yarkivaev/source-to-sink';
+import userDecisionCodec from '../codecs/userDecisionCodec.js';
+import userDecisionSink from '../sinks/userDecisionSink.js';
+
+/**
+ * Pipeline for logging operator tag decisions from RabbitMQ to PostgreSQL.
+ *
+ * Subscribes to the user_decisions STOMP exchange and inserts each received
+ * decision into the user_decisions audit table. Decisions carry machine,
+ * start timestamp, operator identity, and full JSON payload.
+ *
+ * @example
+ *   const pipeline = decisionPipeline(
+ *     'stomp://rabbitmq:61613',
+ *     'postgresql://scada:scada@postgres/scada',
+ *     { login: 'guest', passcode: 'guest', host: '/' }
+ *   );
+ *   pipeline.start();
+ *
+ * @param {string} stomp - STOMP broker URL
+ * @param {string} postgres - PostgreSQL connection URL
+ * @param {object} config - Pipeline configuration
+ * @returns {object} Pipeline with start() and stop() methods
+ */
+export default function decisionPipeline(stomp, pool, config) {
+    const sink = userDecisionSink(pool);
+    const codec = userDecisionCodec(sink);
+    const source = stompSource(stomp, '/exchange/scada.user_decisions', codec,
+        { login: config.login, passcode: config.passcode, host: config.host });
+    return {
+        /**
+         * Starts consuming user decisions from RabbitMQ.
+         */
+        start() {
+            source.start();
+        },
+        /**
+         * Stops consuming and releases resources.
+         */
+        stop() {
+            source.stop();
+            pool.end();
+        }
+    };
+}

package/src/infrastructure/ingest/pipelines/segmentPipeline.js

@@ -0,0 +1,60 @@
+import {
+    postgresSink,
+    stompSource
+} from '@yarkivaev/source-to-sink';
+import segmentDispatch from '../../../domain/segment/dispatch.js';
+import silenceBudget from '../../../domain/segment/silenceBudget.js';
+import segmentCodec from '../codecs/segmentCodec.js';
+import silentOpenWatch from '../silentOpenWatch.js';
+import closeOrphanOpen from '../sinks/closeOrphanOpen.js';
+import closeSilentOpen from '../sinks/closeSilentOpen.js';
+import retagSink from '../sinks/retagSink.js';
+
+export const segmentColumns = ['machine', 'name', 'start_time', 'end_time', 'duration',
+                               'options', 'tags', 'properties', 'resolved'];
+export const segmentConflict = ['machine', 'start_time'];
+export const segmentUpdateColumns = ['name', 'end_time', 'duration', 'options', 'resolved'];
+export const splitUpdateColumns = ['name', 'end_time', 'duration', 'tags', 'options', 'resolved'];
+export const segmentsIngestDestination = '/queue/scada.segments.ingest';
+
+export { default as segmentDispatch } from '../../../domain/segment/dispatch.js';
+
+/**
+ * Pipeline for streaming STOMP segment data to PostgreSQL segments table.
+ * Subscribes to the durable ingest queue so shovel traffic survives consumer gaps.
+ *
+ * @param {string} stomp - STOMP broker URL
+ * @param {string} postgres - PostgreSQL connection URL
+ * @param {object} pool - pg pool
+ * @param {object} config - Pipeline configuration
+ * @returns {object} Pipeline with start() and stop() methods
+ */
+export default function segmentPipeline(stomp, postgres, pool, config) {
+    const segmentSink = postgresSink(postgres, 'segments', segmentColumns,
+        { conflict: segmentConflict, update: segmentUpdateColumns });
+    const splitSink = postgresSink(postgres, 'segments', segmentColumns,
+        { conflict: segmentConflict, update: splitUpdateColumns });
+    const retag = retagSink(pool);
+    const closer = closeOrphanOpen(pool);
+    const dispatch = segmentDispatch(segmentSink, retag, splitSink, closer);
+    const codec = segmentCodec(dispatch);
+    const destination = config.segmentsDestination || segmentsIngestDestination;
+    const source = stompSource(stomp, destination, codec,
+        { login: config.login, passcode: config.passcode, host: config.host });
+    const window = config.segmentWindow || 15;
+    const budget = silenceBudget(window);
+    const pollMs = (config.poll || 5) * 1000;
+    const silence = silentOpenWatch(closeSilentOpen(pool), budget, { intervalMs: pollMs });
+    return {
+        destination,
+        start() {
+            source.start();
+            void silence.start();
+        },
+        stop() {
+            silence.stop();
+            source.stop();
+            pool.end();
+        }
+    };
+}

package/src/infrastructure/ingest/processingErrorLog.js

@@ -0,0 +1,25 @@
+/**
+ * Emits a structured processing error line to stderr.
+ *
+ * Keeps payloads compact to avoid flooding logs with huge messages.
+ *
+ * @param {string} stage - Processing stage identifier
+ * @param {Error} error - Original processing error
+ * @param {object} context - Optional diagnostic context
+ */
+export default function processingErrorLog(stage, error, context = {}) {
+    const safe = {};
+    for (const [key, value] of Object.entries(context)) {
+        if (typeof value === 'string' && value.length > 500) {
+            safe[key] = `${value.slice(0, 500)}...[truncated]`;
+        } else {
+            safe[key] = value;
+        }
+    }
+    const payload = {
+        stage,
+        message: error?.message || String(error),
+        context: safe
+    };
+    process.stderr.write(`[supervisor-sink] processing error ${JSON.stringify(payload)}\n`);
+}

package/src/infrastructure/ingest/silentOpenWatch.js

@@ -0,0 +1,50 @@
+/**
+ * Periodic wall-clock sweep that closes stale open segments without waiting for STOMP.
+ *
+ * @param {object} closer - closeSilentOpen result with close(budgetSeconds)
+ * @param {number} budget - silence budget in seconds
+ * @param {object} options - intervalMs for sweep period
+ * @returns {object} watch with start() and stop()
+ *
+ * @example
+ *   const watch = silentOpenWatch(closer, 30, { intervalMs: 5000 });
+ *   await watch.start();
+ */
+export default function silentOpenWatch(closer, budget, options) {
+    if (!closer || typeof closer.close !== 'function') {
+        throw new Error('Closer must have a close() method');
+    }
+    if (typeof budget !== 'number' || !(budget > 0)) {
+        throw new Error(`Budget must be a positive number: ${budget}`);
+    }
+    const intervalMs = options && options.intervalMs ? options.intervalMs : 5000;
+    let timer;
+    let active = false;
+    async function tick() {
+        try {
+            await closer.close(budget);
+        } catch {
+            /* next tick retries */
+        }
+    }
+    return {
+        async start() {
+            if (active) {
+                return;
+            }
+            active = true;
+            await tick();
+            timer = setInterval(() => {
+                void tick();
+            }, intervalMs);
+        },
+        stop() {
+            if (!active) {
+                return;
+            }
+            active = false;
+            clearInterval(timer);
+            timer = undefined;
+        }
+    };
+}

package/src/infrastructure/ingest/sinks/alertSink.js

@@ -0,0 +1,43 @@
+import { alertIngestAction } from '../../../domain/alerting/ingest.js';
+import processingErrorLog from '../processingErrorLog.js';
+
+/**
+ * PostgreSQL sink for alert records with INSERT/UPDATE branching.
+ *
+ * @param {object} pool - pg.Pool instance with query() method
+ * @returns {object} Sink with accept() and stop() methods
+ *
+ * @example
+ *   const sink = alertSink(pool);
+ *   await sink.accept({ name: 'low_cosphi', message: 'msg', machine: 'm2', severity: 'warning', status: 'pending', timestamp: '2023-11-14T22:13:20.000Z' });
+ */
+export default function alertSink(pool) {
+    return {
+        async accept(record) {
+            try {
+                const action = alertIngestAction(record);
+                if (action === 'insert') {
+                    await pool.query(
+                        'INSERT INTO alerts (name, message, machine, severity, timestamp) VALUES ($1, $2, $3, $4, $5)',
+                        [record.name, record.message, record.machine, record.severity, record.timestamp]
+                    );
+                } else if (action === 'acknowledge') {
+                    await pool.query(
+                        'UPDATE alerts SET acknowledged = TRUE WHERE name = $1 AND machine = $2 AND acknowledged = FALSE',
+                        [record.name, record.machine]
+                    );
+                }
+            } catch (error) {
+                processingErrorLog('alert_sink_write', error, {
+                    name: record.name,
+                    machine: record.machine,
+                    status: record.status
+                });
+                throw error;
+            }
+        },
+        async stop() {
+            await pool.end();
+        }
+    };
+}

package/src/infrastructure/ingest/sinks/closeOrphanOpen.js

@@ -0,0 +1,32 @@
+import processingErrorLog from '../processingErrorLog.js';
+
+/**
+ * Closes stale open segment rows for one machine before a new open segment lands.
+ *
+ * @param {object} pool - pg Pool with query() method
+ * @returns {object} closer with close(machine, startTime)
+ *
+ * @example
+ *   const closer = closeOrphanOpen(pool);
+ *   await closer.close('m1', '2024-01-01T00:00:00.000Z');
+ */
+export default function closeOrphanOpen(pool) {
+    if (!pool || typeof pool.query !== 'function') {
+        throw new Error('Pool must have a query() method');
+    }
+    return {
+        async close(machine, startTime) {
+            try {
+                await pool.query(
+                    `UPDATE segments
+                     SET end_time = start_time + interval '1 second', duration = 1
+                     WHERE machine = $1 AND duration = 0 AND start_time <> $2`,
+                    [machine, startTime]
+                );
+            } catch (error) {
+                processingErrorLog('close_orphan_open', error, { machine, startTime });
+                throw error;
+            }
+        }
+    };
+}

package/src/infrastructure/ingest/sinks/closeSilentOpen.js

@@ -0,0 +1,39 @@
+import processingErrorLog from '../processingErrorLog.js';
+
+/**
+ * Closes open segment rows whose advanced end boundary is older than a wall-clock silence budget.
+ * Requires end_time > start_time so a lone Started (end equals start) is never treated as silence.
+ * Does not insert a replacement Started row.
+ *
+ * @param {object} pool - pg Pool with query() method
+ * @returns {object} closer with close(budgetSeconds)
+ *
+ * @example
+ *   const closer = closeSilentOpen(pool);
+ *   await closer.close(30);
+ */
+export default function closeSilentOpen(pool) {
+    if (!pool || typeof pool.query !== 'function') {
+        throw new Error('Pool must have a query() method');
+    }
+    return {
+        async close(budgetSeconds) {
+            try {
+                await pool.query(
+                    `UPDATE segments
+                     SET duration = GREATEST(
+                           1,
+                           EXTRACT(EPOCH FROM (end_time - start_time))
+                         )
+                     WHERE duration = 0
+                       AND end_time > start_time
+                       AND end_time < NOW() - make_interval(secs => $1)`,
+                    [budgetSeconds]
+                );
+            } catch (error) {
+                processingErrorLog('close_silent_open', error, { budgetSeconds });
+                throw error;
+            }
+        }
+    };
+}