@yarkivaev/scada 2.3.115 → 2.4.0

package/src/infrastructure/streams/amqpSource.js

@@ -0,0 +1,97 @@
+import amqpSession from './amqpSource/session.js';
+import budgetAccept from './budgetAccept.js';
+
+function capacity(options) {
+    const settings = { prefetch: 32, maxMessageBytes: 1048576, ...options };
+    settings.maxPendingBytes ??= settings.prefetch * settings.maxMessageBytes;
+    ['prefetch', 'maxMessageBytes', 'maxPendingBytes'].forEach((name) => {
+        if (!Number.isSafeInteger(settings[name]) || settings[name] < 1) { throw new TypeError(`${name} must be a positive safe integer`); }
+    });
+    return settings;
+}
+
+function deliveryFor(channel, message, options) {
+    let settled = false;
+    return {
+        content: message.content, fields: message.fields, properties: message.properties,
+        ack() {
+            if (!settled && options.active()) { channel.ack(message); settled = true; }
+        },
+        reject(requeue = true) {
+            if (!settled && options.active()) { channel.nack(message, false, requeue); settled = true; }
+        }
+    };
+}
+function reportFailure(options, error) {
+    const report = options.onError || options.report;
+    try {
+        if (report) { report(error); }
+        else { process.stderr.write(`amqp delivery failed ${error.message}\n`); }
+    } catch {
+        // Error reporting must not reject a broker callback.
+    }
+}
+function rejectDelivery(delivery, options, error) {
+    try { delivery.reject(true); }
+    catch (settlementError) { reportFailure(options, settlementError); }
+    reportFailure(options, error);
+}
+function consume(channel, collector, options) {
+    return async (message) => {
+        if (!message) { channel.emit('close'); return; }
+        const delivery = deliveryFor(channel, message, options);
+        try {
+            await budgetAccept(collector, delivery, message.content.length, options);
+            if (options.ackMode !== 'manual') { delivery.ack(); }
+        } catch (error) {
+            rejectDelivery(delivery, options, error);
+        }
+    };
+}
+function trackedConsume(channel, collector, options, pending) {
+    const accept = consume(channel, collector, options);
+    const state = { bytes: 0 };
+    return (message) => {
+        if (!options.active()) { return Promise.resolve(); }
+        const bytes = message?.content?.length ?? 0;
+        if (bytes > options.maxMessageBytes || state.bytes + bytes > options.maxPendingBytes) {
+            try { channel.nack(message, false, false); } catch (error) { reportFailure(options, error); }
+            reportFailure(options, new Error('AMQP payload capacity exceeded; rejected without requeue'));
+            return Promise.resolve();
+        }
+        state.bytes += bytes;
+        const delivery = accept(message);
+        pending.add(delivery);
+        return delivery.finally(() => { state.bytes -= bytes; pending.delete(delivery); });
+    };
+}
+async function topology(channel, queue, options) {
+    await Promise.all((options.exchanges || []).map((exchange) => {
+        return channel.assertExchange(exchange.name, exchange.type, exchange.options);
+    }));
+    const declared = await channel.assertQueue(queue, options.queueOptions || { durable: true });
+    const name = declared?.queue || queue;
+    await Promise.all((options.bindings || []).map((binding) => {
+        return channel.bindQueue(name, binding.exchange, binding.pattern, binding.args);
+    }));
+    await channel.prefetch(options.prefetch || 32);
+    return name;
+}
+/** Delivers AMQP records, acknowledging only after collector acceptance. */
+export default function amqpSource(url, queue, collector, options = {}) {
+    if (!collector || typeof collector.accept !== 'function') { throw new Error('Collector must have an accept() method'); }
+    const settings = capacity(options);
+    const pending = new Set();
+    const state = {};
+    const session = amqpSession(url, async (channel) => {
+        const generation = state.session.generation();
+        const config = { ...settings, active() {
+            return state.session.active() && state.session.generation() === generation;
+        } };
+        const name = await topology(channel, queue, settings);
+        const result = await channel.consume(name, trackedConsume(channel, collector, config, pending), { noAck: false });
+        return result.consumerTag;
+    }, { ...settings, drain() { return Promise.all([...pending]); } });
+    state.session = session;
+    return session;
+}

package/src/infrastructure/streams/brokerClose.js

@@ -0,0 +1,19 @@
+import deadline from './deadline.js';
+
+/** Attempt every cleanup stage; an uncertain stage prevents reuse even if later cleanup succeeds. */
+export default async function brokerClose(actions, timeoutMs, label, destroy) {
+    const failures = [];
+    for (const action of actions) {
+        try {
+            // Cleanup order matters, but one hung stage must not skip transport closure.
+            // eslint-disable-next-line no-await-in-loop
+            await deadline(Promise.resolve().then(action), timeoutMs, label, destroy);
+        } catch (error) {
+            if (error.name !== 'IllegalOperationError') { failures.push(error); }
+        }
+    }
+    if (failures.length) { throw failures[0]; }
+}
+export function destroyConnection(connection) {
+    connection?.connection?.stream?.destroy();
+}

package/src/infrastructure/streams/budgetAccept.js

@@ -0,0 +1,6 @@
+/** Budget leases belong to actual acceptance, never to a timeout wrapper. */
+export default async function budgetAccept(collector, record, bytes, options, context) {
+    const release = options.budget?.reserve({ bytes, records: 1 });
+    try { return await collector.accept(record, context); }
+    finally { release?.(); }
+}

package/src/infrastructure/streams/clickhouseSink.js

@@ -0,0 +1,48 @@
+import { createClient } from '@clickhouse/client';
+import deadline from './deadline.js';
+import positive from './positive.js';
+
+/**
+ * ClickHouse sink for batch record insertion.
+ *
+ * Creates a ClickHouse client internally and implements the Sink
+ * interface for use with batch collectors.
+ *
+ * @example
+ * const sink = clickhouseSink('http://localhost:8123', 'metrics');
+ * await sink.write([{ ts: Date.now(), value: 42 }]);
+ *
+ * @param {string} url - ClickHouse URL (e.g., 'http://localhost:8123')
+ * @param {string} table - Target table name
+ * @returns {object} Sink with write(records) method
+ */
+export default function clickhouseSink(url, table, options = {}) {
+  if (typeof url !== 'string' || url.length === 0) {
+    throw new Error('URL must be a non-empty string');
+  }
+  if (typeof table !== 'string' || table.length === 0) {
+    throw new Error('Table must be a non-empty string');
+  }
+  positive(options.stopTimeoutMs ?? 5000, 'stopTimeoutMs');
+  const client = (options.createClient || createClient)({ url });
+  let stopping;
+  return {
+    /**
+     * Writes records to ClickHouse table.
+     *
+     * @param {Array} records - Array of records to insert
+     * @returns {Promise} Promise resolving when insert completes
+     */
+    write(records, { signal } = {}) {
+      if (stopping) { return Promise.reject(new Error('ClickHouse sink stopped')); }
+      if (signal?.aborted) { return Promise.reject(signal.reason || new Error('ClickHouse write aborted')); }
+      // Native ClickHouse API uses snake_case for cancellation.
+       
+      return client.insert({ table, values: records, format: 'JSONEachRow', abort_signal: signal });
+    },
+    stop() {
+      stopping ??= deadline(Promise.resolve().then(() => { return client.close(); }), options.stopTimeoutMs ?? 5000, 'ClickHouse close');
+      return stopping;
+    }
+  };
+}

package/src/infrastructure/streams/deadline.js

@@ -0,0 +1 @@
+export { deadline as default } from '@yarkivaev/source-to-sink';

package/src/infrastructure/streams/follow.js

@@ -0,0 +1,41 @@
+/**
+ * Continues a delivery chain after a rejected prior step.
+ *
+ * @param {Promise} pending - prior chain
+ * @returns {Promise} settled chain
+ */
+function resume(pending) {
+    return pending.then((value) => {
+        return value;
+    }, (error) => {
+        return error;
+    });
+}
+
+/**
+ * Chains one arrival: read the body, then start deliver.
+ * Serial waits for deliver. Otherwise persist may overlap after enqueue.
+ *
+ * @param {Promise} pending - prior arrival
+ * @param {Function} read - returns a promise for the body
+ * @param {Function} deliver - starts persist for one body
+ * @param {boolean} serial - wait for persist before the next arrival
+ * @returns {Promise} chain
+ *
+ * @example
+ *   let pending = Promise.resolve();
+ *   pending = follow(pending, () => Promise.resolve('a'), write, false);
+ */
+export default function follow(pending, read, deliver, serial) {
+    return resume(pending).then(async () => {
+        const body = await read();
+        const run = deliver(body);
+        if (serial) {
+            return run;
+        }
+        void run.catch((error) => {
+            return error;
+        });
+        return undefined;
+    });
+}

package/src/infrastructure/streams/index.js

@@ -0,0 +1,15 @@
+/** Protocol and storage adapters composed with source-to-sink primitives. */
+export { default as clickhouseSink } from './clickhouseSink.js';
+export { default as postgresSink } from './postgresSink.js';
+export { default as mqttSource } from './mqttSource.js';
+export { default as stompSource } from './stompSource.js';
+export { default as stompSend } from './stompSend.js';
+export { default as lokiSource } from './lokiSource.js';
+export { default as modbusSource, modbusRtuSource } from './modbusSource.js';
+export { default as modbusRtuBusSource } from './modbusRtuBusSource.js';
+export { default as mqttSink } from './mqttSink.js';
+export { default as sqliteSink } from './sqliteSink.js';
+export { default as amqpSink } from './amqpSink.js';
+export { default as amqpSource } from './amqpSource.js';
+export { default as amqpSession } from './amqpSource/session.js';
+export { default as serialSource } from './serialSource.js';

package/src/infrastructure/streams/lokiSource.js

@@ -0,0 +1,88 @@
+import pollingSource from './pollingSource.js';
+import positive from './positive.js';
+
+function config(options) {
+  const settings = { requestTimeoutMs: 5000, maxResponseBytes: 1048576, pageLimit: 1000, queryLagMs: 1000, ...options };
+  ['requestTimeoutMs', 'maxResponseBytes', 'pageLimit'].forEach((key) => { positive(settings[key], key); });
+  if (!Number.isSafeInteger(settings.pageLimit) || settings.pageLimit > 10000 || !Number.isSafeInteger(settings.maxResponseBytes)) {
+    throw new TypeError('Invalid Loki response capacity');
+  }
+  if (!Number.isSafeInteger(settings.queryLagMs) || settings.queryLagMs < 0) { throw new TypeError('Invalid Loki queryLagMs'); }
+  return settings;
+}
+async function body(response, maxBytes) {
+  if (!response.ok) { throw new Error(`Loki HTTP status ${response.status}`); }
+  if (!response.body) { throw new Error('Loki response body missing'); }
+  const chunks = [];
+  let bytes = 0;
+  for await (const chunk of response.body) {
+    bytes += chunk.byteLength;
+    if (bytes > maxBytes) { throw new Error('Loki response capacity exceeded'); }
+    chunks.push(chunk);
+  }
+  return JSON.parse(Buffer.concat(chunks, bytes).toString());
+}
+function entries(data, limit) {
+  if (data.status !== 'success' || !Array.isArray(data.data?.result) || data.data.resultType !== 'streams') {
+    throw new Error('Invalid Loki stream response');
+  }
+  const rows = data.data.result.flatMap((stream) => {
+    if (!Array.isArray(stream.values)) { throw new Error('Invalid Loki stream values'); }
+    return stream.values;
+  });
+  // A full page may hide older/equal-timestamp rows. Do not deliver or advance a partial window.
+  if (rows.length >= limit) { throw new Error('Loki query page saturated; history incomplete, cursor unchanged'); }
+  return rows.map(([ts, line]) => {
+    if (typeof ts !== 'string' || !/^\d+$/u.test(ts) || typeof line !== 'string') { throw new Error('Invalid Loki log entry'); }
+    return { ts: Number(ts) / 1000000, line };
+  });
+}
+function requestUrl(url, query, range, limit) {
+  const params = new URLSearchParams({ query, start: (range.since * 1000000).toString(),
+    end: (range.until * 1000000).toString(), limit: String(limit), direction: 'forward' });
+  return `${url}/loki/api/v1/query_range?${params}`;
+}
+async function request(url, settings, signal) {
+  const controller = new AbortController();
+  function abort() { controller.abort(signal.reason); }
+  if (signal?.aborted) { abort(); }
+  signal?.addEventListener('abort', abort, { once: true });
+  const timer = setTimeout(() => { controller.abort(new Error('Loki request deadline exceeded')); }, settings.requestTimeoutMs);
+  try {
+    const response = await globalThis.fetch(url, { signal: controller.signal });
+    return entries(await body(response, settings.maxResponseBytes), settings.pageLimit);
+  } catch (error) {
+    controller.abort(error);
+    try { settings.onError?.(error); } catch {
+      // Diagnostics cannot suppress the failed read or change its cursor.
+    }
+    throw error;
+  } finally {
+    clearTimeout(timer);
+    signal?.removeEventListener('abort', abort);
+  }
+}
+/**
+ * Bounded Loki polling. Saturated pages fail without advancing the in-memory cursor.
+ * Windows lag wall-clock time by queryLagMs (default 1s) to allow ingestion visibility.
+ * Entries becoming visible later than this configured lag may still be missed.
+ * This live polling source does not expose portable history checkpoints or pagination.
+ *
+ * @param {string} url - Loki base URL
+ * @param {string} query - LogQL expression
+ * @param {number} interval - Polling interval in seconds
+ * @param {object} collector - Collector with accept()
+ * @param {object} clk - Clock with millis()
+ * @param {object} options - requestTimeoutMs, maxResponseBytes, pageLimit, queryLagMs, onError
+ * @returns {object} Source with async start/stop
+ */
+// Sixth optional argument preserves the established five-argument source API.
+// eslint-disable-next-line max-params
+export default function lokiSource(url, query, interval, collector, clk, options = {}) {
+  if (typeof url !== 'string' || url.length === 0) { throw new Error('URL must be a non-empty string'); }
+  if (typeof query !== 'string' || query.length === 0) { throw new Error('Query must be a non-empty string'); }
+  const settings = config(options);
+  return pollingSource((since, until, { signal } = {}) => {
+    return request(requestUrl(url, query, { since: since - settings.queryLagMs, until: until - settings.queryLagMs }, settings.pageLimit), settings, signal);
+  }, interval, collector, clk);
+}

package/src/infrastructure/streams/modbusConnection.js

@@ -0,0 +1,173 @@
+import ModbusRTU from 'modbus-serial';
+import deadline from './deadline.js';
+
+/**
+ * Idle state for Modbus connection.
+ *
+ * @returns {object} State with no open attempt
+ */
+export function idle() {
+  return {
+    connected() {
+      return false;
+    },
+    retrying() {
+      return false;
+    },
+    opening() {
+      return false;
+    }
+  };
+}
+
+/**
+ * Retrying state for Modbus connection.
+ *
+ * @param {object} handle - Timer handle from setTimeout
+ * @returns {object} State with retrying() returning true
+ */
+function retrying(handle) {
+  return {
+    connected() {
+      return false;
+    },
+    retrying() {
+      return true;
+    },
+    opening() {
+      return false;
+    },
+    cancel() {
+      clearTimeout(handle);
+    }
+  };
+}
+
+/**
+ * In-flight TCP or RTU connect state.
+ *
+ * @returns {object} State with opening() returning true
+ */
+export function opening() {
+  return {
+    connected() {
+      return false;
+    },
+    retrying() {
+      return false;
+    },
+    opening() {
+      return true;
+    }
+  };
+}
+
+/**
+ * Connected state for Modbus connection.
+ *
+ * @param {object} client - ModbusRTU client
+ * @returns {object} State with connected() returning true
+ */
+function connected(client) {
+  return {
+    connected() {
+      return true;
+    },
+    retrying() {
+      return false;
+    },
+    opening() {
+      return false;
+    },
+    disconnect() {
+      client.close();
+    }
+  };
+}
+
+/**
+ * Destroys a Modbus client without throwing if already closed.
+ *
+ * @param {object} client - ModbusRTU client
+ */
+export function drop(client) {
+  if (client && typeof client.destroy === 'function') {
+    client.destroy(() => {
+      return undefined;
+    });
+  }
+}
+
+/**
+ * Builds a ModbusRTU client for one connect attempt.
+ *
+ * @returns {object} Fresh ModbusRTU client
+ */
+export function create() {
+  return new ModbusRTU();
+}
+
+/**
+ * Starts one connect attempt and ignores results after stop().
+ *
+ * @param {object} bag - Mutable lifecycle bag
+ */
+export function attempt(bag) {
+  const token = bag.epoch;
+  bag.state = opening();
+  drop(bag.client);
+  const client = bag.create();
+  bag.client = client;
+  const connecting = bag.connect(client);
+  void connecting.then(() => { if (token !== bag.epoch) { drop(client); } }).catch(() => { return undefined; });
+  bag.opened = deadline(connecting, 5000, 'Modbus connect', (error) => {
+    bag.fatal = error; bag.epoch += 1; drop(client);
+  }).then(() => {
+    if (token !== bag.epoch) {
+      drop(client);
+      return;
+    }
+    bag.delay = bag.interval;
+    bag.state = connected(client);
+    client.setTimeout(bag.interval * 1000);
+    client.once('close', () => {
+      if (token === bag.epoch && bag.client === client) { bag.lost(); }
+    });
+    void bag.source.start().catch((error) => { bag.fatal = error; drop(client); });
+  }).catch((err) => {
+    if (token !== bag.epoch) {
+      return;
+    }
+    drop(bag.client);
+    bag.log.error(`Connection to ${bag.target} failed, retrying in ${bag.delay}s: ${err.message}`);
+    const handle = setTimeout(() => {
+      attempt(bag);
+    }, bag.delay * 1000);
+    bag.state = retrying(handle);
+    bag.delay = Math.min(bag.delay * 2, bag.maxDelay);
+  });
+}
+
+/**
+ * Resets a live session and retries through attempt() backoff.
+ *
+ * @param {object} bag - Mutable lifecycle bag
+ */
+export function resume(bag) {
+  if (!bag.state.connected()) {
+    return;
+  }
+  const drained = bag.source.stop();
+  const { client } = bag;
+  bag.client = undefined;
+  bag.state = idle();
+  drop(client);
+  bag.log.error(`Connection to ${bag.target} lost, retrying in ${bag.delay}s`);
+  const token = bag.epoch;
+  const handle = setTimeout(() => {
+    void drained.then(() => { if (token === bag.epoch) { attempt(bag); } }).catch((error) => { bag.fatal = error; });
+  }, bag.delay * 1000);
+  bag.state = retrying(handle);
+  bag.delay = Math.min(bag.delay * 2, bag.maxDelay);
+}
+

package/src/infrastructure/streams/modbusRtuBusSource.js

@@ -0,0 +1,124 @@
+import ModbusRTU from 'modbus-serial';
+import positive from './positive.js';
+import deadline from './deadline.js';
+
+function validate(path, serial, slaves, poll) {
+    if (typeof path !== 'string' || !path) { throw new Error('Serial path must be a non-empty string'); }
+    if (!serial || !Number.isFinite(serial.baudRate) || serial.baudRate <= 0) { throw new Error('Serial baudRate must be a positive number'); }
+    if (!Array.isArray(slaves) || !slaves.length) { throw new Error('Slaves must be a non-empty array'); }
+    positive(poll.interval, 'interval');
+    slaves.forEach((slave) => {
+        if (!Number.isInteger(slave.slaveId) || slave.slaveId < 1 || slave.slaveId > 247 || typeof slave.collector?.accept !== 'function') {
+            throw new TypeError('Each RTU slave requires an ID from 1 to 247 and a collector');
+        }
+    });
+}
+function report(ctx, error) {
+    try { ctx.poll.log?.error(`RTU bus on ${ctx.path}: ${error.message}`); } catch {
+        // Diagnostics cannot stop recovery.
+    }
+}
+function close(ctx) {
+    ctx.closing ??= deadline(new Promise((resolve, reject) => {
+        ctx.client.close((error) => { if (error) { reject(error); } else { resolve(); } });
+    }), ctx.poll.stopTimeoutMs, 'RTU close');
+    return ctx.closing;
+}
+function pause(ctx) {
+    return new Promise((resolve) => { ctx.wake = resolve; ctx.timer = setTimeout(resolve, ctx.poll.interval * 1000); })
+        .finally(() => { clearTimeout(ctx.timer); ctx.wake = null; });
+}
+async function read(ctx, slave) {
+    ctx.client.setID(slave.slaveId);
+    const result = await deadline(ctx.client.readHoldingRegisters(ctx.poll.address, ctx.poll.count), ctx.poll.readTimeoutMs, 'RTU read');
+    if (ctx.active) {
+        await deadline(Promise.resolve().then(() => {
+            return slave.collector.accept(result.data, { signal: ctx.controller.signal });
+        }), ctx.poll.acceptTimeoutMs, 'RTU acceptance', (error) => { ctx.controller.abort(error); });
+    }
+}
+async function cycle(ctx) {
+    for (const slave of ctx.slaves) {
+        if (!ctx.active) { return; }
+        try {
+            // Shared setID requires every read and acceptance to finish before the next slave.
+            // eslint-disable-next-line no-await-in-loop
+            await read(ctx, slave);
+        } catch (error) {
+            if (error.code === 'OPERATION_TIMEOUT') { throw error; }
+            report(ctx, error);
+        }
+    }
+}
+async function session(ctx) {
+    ctx.opening = true;
+    const opening = ctx.client.connectRTUBuffered(ctx.path, ctx.serial);
+    void Promise.resolve(opening).then(() => {
+        ctx.opening = false;
+        if (!ctx.active) { return close(ctx); }
+        return undefined;
+    }, () => { ctx.opening = false; }).catch((error) => { report(ctx, error); });
+    await deadline(opening, ctx.poll.connectTimeoutMs, 'RTU connect');
+    while (ctx.active) {
+        // eslint-disable-next-line no-await-in-loop
+        await cycle(ctx);
+        // eslint-disable-next-line no-await-in-loop
+        if (ctx.active) { await pause(ctx); }
+    }
+}
+async function run(ctx) {
+    while (ctx.active) {
+        ctx.client = (ctx.poll.createClient || (() => { return new ModbusRTU(); }))();
+        ctx.client.setTimeout(ctx.poll.readTimeoutMs);
+        ctx.closing = null;
+        try {
+            // eslint-disable-next-line no-await-in-loop
+            await session(ctx);
+        } catch (error) {
+            report(ctx, error);
+            if (error.code === 'OPERATION_TIMEOUT') { Object.assign(ctx, { fatal: error, active: false }); }
+        } finally {
+            // Never treat closing a still-opening handle as proof its eventual port is closed.
+            // eslint-disable-next-line no-await-in-loop
+            if (!ctx.opening) { await close(ctx); }
+        }
+        // eslint-disable-next-line no-await-in-loop
+        if (ctx.active) { await pause(ctx); }
+    }
+}
+function launch(path, serial, slaves, poll) {
+    const ctx = { path, serial, slaves, poll, active: true, controller: new AbortController() };
+    ctx.task = run(ctx);
+    void ctx.task.catch((error) => { ctx.fatal = error; ctx.active = false; report(ctx, error); });
+    return ctx;
+}
+/** One serialized RTU bus. stop drains/closes; uncertain deadlines quarantine this instance. */
+export default function modbusRtuBusSource(path, serial, slaves, poll) {
+    validate(path, serial, slaves, poll);
+    const settings = { readTimeoutMs: 5000, connectTimeoutMs: 5000, acceptTimeoutMs: 10000, stopTimeoutMs: 5000, ...poll };
+    ['readTimeoutMs', 'connectTimeoutMs', 'acceptTimeoutMs', 'stopTimeoutMs'].forEach((key) => { positive(settings[key], key); });
+    let ctx;
+    let revision = 0;
+    return {
+        async start() {
+            if (ctx?.active) { return; }
+            revision += 1;
+            const requested = revision;
+            if (ctx) {
+                await ctx.task;
+                if (ctx.fatal) { throw ctx.fatal; }
+            }
+            if (requested === revision) { ctx = launch(path, serial, slaves, settings); }
+        },
+        stop() {
+            revision += 1;
+            if (!ctx) { return Promise.resolve(); }
+            ctx.active = false;
+            ctx.controller.abort(new Error('RTU source stopped'));
+            ctx.wake?.();
+            return deadline(ctx.task, settings.stopTimeoutMs, 'RTU stop', (error) => { ctx.fatal = error; if (!ctx.opening) { void close(ctx).catch(() => { return undefined; }); } }).then(() => {
+                if (ctx.fatal) { throw ctx.fatal; }
+            });
+        }
+    };
+}