@yarkivaev/scada 2.3.114 → 2.3.116

package/src/infrastructure/ingest/durableConsume.js

@@ -1,169 +1,2 @@
-import amqp from 'amqplib';
-
-function connectBroker(url) {
-    return amqp.connect(url, { heartbeat: 10 });
-}
-
-function scheduleLater(fn, pause) {
-    return setTimeout(fn, pause);
-}
-
-function clearLater(handle) {
-    clearTimeout(handle);
-}
-
-function reportFailure(error) {
-    process.stderr.write(`amqp consume failed ${error.message}\n`);
-}
-
-function nextDelay(delay) {
-    return Math.min(delay * 2, 30000);
-}
-
-function blankState() {
-    return {
-        generation: 0,
-        timer: undefined,
-        stopping: false,
-        delay: 1000,
-        session: undefined
-    };
-}
-
-function running(state) {
-    return Boolean(state.session) || (state.generation > 0 && !state.stopping);
-}
-
-function scheduleRetry(ctx, generation) {
-    if (ctx.state.stopping || generation !== ctx.state.generation) {
-        return;
-    }
-    const pause = ctx.state.delay;
-    ctx.state.delay = nextDelay(ctx.state.delay);
-    ctx.state.timer = ctx.schedule(() => {
-        ctx.state.timer = undefined;
-        return ctx.open(generation);
-    }, pause);
-}
-
-function bindClose(ctx, generation, live) {
-    let dropped = false;
-    function lose() {
-        if (dropped || ctx.state.stopping || generation !== ctx.state.generation) {
-            return;
-        }
-        dropped = true;
-        if (ctx.state.session === live) {
-            ctx.state.session = undefined;
-        }
-        scheduleRetry(ctx, generation);
-    }
-    live.connection.on('error', lose);
-    live.connection.on('close', lose);
-    live.channel.on('error', lose);
-    live.channel.on('close', lose);
-}
-
-async function accept(ctx, generation, connection) {
-    if (ctx.state.stopping || generation !== ctx.state.generation) {
-        await connection.close();
-        return null;
-    }
-    return connection.createChannel();
-}
-
-async function keep(ctx, generation, connection, channel, tag) {
-    if (ctx.state.stopping || generation !== ctx.state.generation) {
-        await channel.close();
-        await connection.close();
-        return false;
-    }
-    const live = { connection, channel, tag };
-    ctx.state.session = live;
-    ctx.state.delay = 1000;
-    bindClose(ctx, generation, live);
-    return true;
-}
-
-async function open(ctx, generation) {
-    if (ctx.state.stopping || generation !== ctx.state.generation) {
-        return;
-    }
-    try {
-        const connection = await ctx.connect(ctx.url);
-        const channel = await accept(ctx, generation, connection);
-        if (!channel) {
-            return;
-        }
-        const tag = await ctx.subscribe(channel);
-        await keep(ctx, generation, connection, channel, tag);
-    } catch (error) {
-        ctx.report(error);
-        scheduleRetry(ctx, generation);
-    }
-}
-
-function begin(ctx) {
-    if (running(ctx.state)) {
-        return Promise.resolve();
-    }
-    ctx.state.stopping = false;
-    ctx.state.generation += 1;
-    ctx.state.delay = 1000;
-    return open(ctx, ctx.state.generation);
-}
-
-async function end(ctx) {
-    ctx.state.stopping = true;
-    ctx.state.generation += 1;
-    if (ctx.state.timer) {
-        ctx.clear(ctx.state.timer);
-        ctx.state.timer = undefined;
-    }
-    const live = ctx.state.session;
-    ctx.state.session = undefined;
-    if (!live) {
-        return;
-    }
-    await live.channel.cancel(live.tag);
-    await live.channel.close();
-    await live.connection.close();
-}
-
-/**
- * Keeps one AMQP consumer subscribed across broker restarts.
- *
- * @param {string} url - RabbitMQ AMQP URL
- * @param {function} subscribe - async (channel) => consumerTag
- * @param {object} [options] - connect, schedule, clear, report overrides
- * @returns {object} session with start and stop
- *
- * @example
- *   const session = durableConsume('amqp://localhost', async (channel) => {
- *       const tag = await channel.consume('scada.telemetry.ingest', () => {});
- *       return tag.consumerTag;
- *   });
- *   await session.start();
- */
-export default function durableConsume(url, subscribe, options = {}) {
-    const ctx = {
-        url,
-        subscribe,
-        connect: options.connect || connectBroker,
-        schedule: options.schedule || scheduleLater,
-        clear: options.clear || clearLater,
-        report: options.report || reportFailure,
-        state: blankState(),
-        open(generation) {
-            return open(ctx, generation);
-        }
-    };
-    return {
-        start() {
-            return begin(ctx);
-        },
-        stop() {
-            return end(ctx);
-        }
-    };
-}
+// Compatibility facade; transport lifecycle belongs to source-to-sink.
+export { amqpSession as default } from '@yarkivaev/source-to-sink';

package/src/infrastructure/ingest/modbus/silentStreams.js

@@ -1,4 +1,4 @@
-import { clock } from '@yarkivaev/source-to-sink';
+import { clock, intervalSource } from '@yarkivaev/source-to-sink';
 
 /**
  * Whether a stream is past the silence budget.
@@ -83,15 +83,12 @@ function gateState(config) {
         intervalSec: interval,
         sources,
         clk: config.clock || clock(),
-        delay: config.delay || ((fn, ms) => {
-            return setInterval(fn, ms);
-        }),
-        clear: config.clear || clearInterval,
+        delay: config.delay || setTimeout,
+        clear: config.clear || clearTimeout,
         lastSeen: new Map(),
         held: new Map(),
         live: new Set(),
         collector: undefined,
-        timer: undefined,
         running: false
     };
 }
@@ -119,6 +116,9 @@ export default function silentStreams(config) {
     function runPulse() {
         pulse(state);
     }
+    const ticker = intervalSource(state.intervalSec * 1000, { accept: runPulse }, {
+        schedule: state.delay, clear: state.clear
+    });
     return {
         seen(name, at = state.clk.millis()) {
             state.lastSeen.set(name, at);
@@ -134,16 +134,14 @@ export default function silentStreams(config) {
             };
         },
         start() {
+            if (state.running) { return; }
             state.running = true;
             runPulse();
-            state.timer = state.delay(runPulse, state.intervalSec * 1000);
+            void ticker.start();
         },
         stop() {
             state.running = false;
-            if (state.timer !== undefined) {
-                state.clear(state.timer);
-                state.timer = undefined;
-            }
+            void ticker.stop();
             for (const name of [...state.live]) {
                 stopOne(state, name);
             }

package/src/infrastructure/ingest/silentOpenWatch.js

@@ -1,3 +1,5 @@
+import { intervalSource } from '@yarkivaev/source-to-sink';
+
 /**
  * Periodic wall-clock sweep that closes stale open segments without waiting for STOMP.
  *
@@ -18,46 +20,7 @@ export default function silentOpenWatch(closer, budget, options) {
         throw new Error(`Budget must be a positive number: ${budget}`);
     }
     const intervalMs = options && options.intervalMs ? options.intervalMs : 5000;
-    const gate = { busy: false };
-    let timer;
-    let active = false;
-    function claim() {
-        const free = !gate.busy;
-        gate.busy = true;
-        return free;
-    }
-    function release() {
-        gate.busy = false;
-    }
-    async function tick() {
-        if (!claim()) {
-            return;
-        }
-        try {
-            await closer.close(budget);
-        } catch {
-            /* next tick retries */
-        }
-        release();
-    }
-    return {
-        async start() {
-            if (active) {
-                return;
-            }
-            active = true;
-            await tick();
-            timer = setInterval(() => {
-                void tick();
-            }, intervalMs);
-        },
-        stop() {
-            if (!active) {
-                return;
-            }
-            active = false;
-            clearInterval(timer);
-            timer = undefined;
-        }
-    };
+    return intervalSource(intervalMs, {
+        accept() { return closer.close(budget); }
+    }, { immediate: true });
 }

package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js

@@ -1,5 +1,4 @@
-import { batch, circuit, clock, timedBatch } from '@yarkivaev/source-to-sink';
-import durableConsume from '../durableConsume.js';
+import { amqpSource, batch, circuit, clock, timedBatch } from '@yarkivaev/source-to-sink';
 import deliverToMqttRecord from './deliverToMqttRecord.js';
 import metricsCodec from '../mqtt/metricsTransformer.js';
 import streamNameFromTopic from './streamNameFromTopic.js';
@@ -20,7 +19,7 @@ export function acceptTelemetryDeliver(codec, fields, content, onSeen) {
             onSeen(name);
         }
     }
-    codec.accept({ topic: record.topic, payload: record.payload.toString() });
+    return codec.accept({ topic: record.topic, payload: record.payload.toString() });
 }
 
 /**
@@ -50,16 +49,6 @@ export function telemetryConsumer(codec, channel, onSeen) {
  * @param {object} [options] - batch and prefetch options
  * @returns {object} ingest with start and stop
  */
-function subscribeMetrics(queue, prefetch, codec, onSeen) {
-    return async (channel) => {
-        await channel.assertQueue(queue, { durable: true });
-        channel.prefetch(prefetch);
-        const onMessage = telemetryConsumer(codec, channel, onSeen);
-        const tag = await channel.consume(queue, onMessage, { noAck: false });
-        return tag.consumerTag;
-    };
-}
-
 export default function amqpMetricsIngest(amqpUrl, queue, sink, options = {}) {
     const prefetch = options.prefetch || 32;
     const size = options.size || 100;
@@ -70,11 +59,11 @@ export default function amqpMetricsIngest(amqpUrl, queue, sink, options = {}) {
     const breaker = circuit(threshold, timeout, clk);
     const collector = timedBatch(batch(sink, size, breaker), interval);
     const codec = metricsCodec(collector);
-    const session = durableConsume(
-        amqpUrl,
-        subscribeMetrics(queue, prefetch, codec, options.onSeen),
-        options
-    );
+    const session = amqpSource(amqpUrl, queue, {
+        accept(delivery) {
+            return acceptTelemetryDeliver(codec, delivery.fields, delivery.content, options.onSeen);
+        }
+    }, { ...options, prefetch });
     return {
         start() {
             return session.start();

package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js

@@ -1,4 +1,4 @@
-import durableConsume from '../durableConsume.js';
+import { amqpSource } from '@yarkivaev/source-to-sink';
 import deliverToMqttRecord from './deliverToMqttRecord.js';
 
 /**
@@ -34,24 +34,14 @@ export function relayConsumer(sink, channel) {
  * @param {object} [options] - prefetch count
  * @returns {object} Relay with start and stop
  */
-function subscribeRelay(queue, prefetch, sink) {
-    return async (channel) => {
-        await channel.assertQueue(queue, { durable: true });
-        channel.prefetch(prefetch);
-        const onMessage = relayConsumer(sink, channel);
-        const tag = await channel.consume(queue, onMessage, { noAck: false });
-        return tag.consumerTag;
-    };
-}
-
 export default function amqpMqttRelay(amqpUrl, queue, sink, options = {}) {
     const prefetch = options.prefetch || 32;
     let opened = false;
-    const session = durableConsume(
-        amqpUrl,
-        subscribeRelay(queue, prefetch, sink),
-        options
-    );
+    const session = amqpSource(amqpUrl, queue, {
+        accept(delivery) {
+            return sink.write([deliverToMqttRecord(delivery.fields, delivery.content)]);
+        }
+    }, { ...options, prefetch });
     return {
         start() {
             if (!opened) {
@@ -62,7 +52,8 @@ export default function amqpMqttRelay(amqpUrl, queue, sink, options = {}) {
         },
         async stop() {
             await session.stop();
-            sink.stop();
+            await sink.stop();
+            opened = false;
         }
     };
 }

package/src/infrastructure/messaging/rabbitmq/definitions.js

@@ -0,0 +1,24 @@
+/** Builds durable topic exchanges, queues and AMQP 0-9-1 outboxes for one virtual host. */
+export default function rabbitmqDefinitions({ vhost, users = [], permissions = [], exchanges = [],
+    queues = [], bindings = [], outboxes = [] }) {
+    if (!vhost) { throw new Error('a virtual host is required'); }
+    const routes = outboxes.flatMap(({ exchange, queue, keys }) => {
+        return keys.map((key) => { return { source: exchange, destination: queue, key }; });
+    });
+    const names = [...queues, ...outboxes.map(({ queue }) => { return queue; })];
+    return { vhosts: [{ name: vhost }], users, permissions,
+        exchanges: exchanges.map((name) => { return { name, vhost, type: 'topic', durable: true, auto_delete: false }; }),
+        queues: [...new Set(names)].map((name) => { return { name, vhost, durable: true, auto_delete: false, arguments: {} }; }),
+        bindings: [...bindings, ...routes].map(({ source, destination, key }) => {
+            return { source, vhost, destination, destination_type: 'queue', routing_key: key, arguments: {} };
+        }),
+        parameters: outboxes.filter(({ shovel }) => { return shovel; }).map(({ queue, shovel }) => {
+            return { vhost, component: 'shovel', name: shovel.name, value: {
+                'src-protocol': 'amqp091', 'src-uri': shovel.sourceUri, 'src-queue': queue,
+                'dest-protocol': 'amqp091', 'dest-uri': shovel.destinationUri,
+                'dest-exchange': shovel.destinationExchange, 'reconnect-delay': shovel.reconnectDelay,
+                'dest-publish-properties': { delivery_mode: 2 }
+            } };
+        })
+    };
+}

package/src/infrastructure/messaging/rabbitmq/reconcile.js

@@ -0,0 +1,104 @@
+/** This module is self-contained so clients can ship it to an existing Node.js pod. */
+function canonical(value) {
+    if (Array.isArray(value)) { return value.map(canonical); }
+    if (value && typeof value === 'object') {
+        return Object.fromEntries(Object.keys(value).sort().map((key) => {
+            return [key, canonical(value[key])];
+        }));
+    }
+    return value;
+}
+
+/** Runtime counters and other fields outside the declaration do not require a write. */
+function same(actual, desired) {
+    return actual && Object.keys(desired).every((key) => {
+        return JSON.stringify(canonical(actual[key])) === JSON.stringify(canonical(desired[key]));
+    });
+}
+
+function apiAt({ base, username, password }) {
+    if (!base || typeof username !== 'string' || typeof password !== 'string') {
+        throw new Error('management base, username and password are required');
+    }
+    const auth = Buffer.from(`${username}:${password}`).toString('base64');
+    return async (method, path, body) => {
+        const response = await fetch(`${base.replace(/\/$/u, '')}/api/${path}`, { method,
+            headers: { authorization: `Basic ${auth}`, 'content-type': 'application/json' },
+            body: body ? JSON.stringify(body) : undefined, signal: AbortSignal.timeout(15000) });
+        if (!response.ok) { throw new Error(`broker ${method} ${path}: HTTP ${response.status}`); }
+        const text = await response.text();
+        return text ? JSON.parse(text) : null;
+    };
+}
+
+/** A conflicting immutable declaration fails in place, preserving the stored queue. */
+async function declare(api, path, desired) {
+    const current = await api('GET', path);
+    const changes = desired.filter((item) => {
+        return !same(current.find((row) => { return row.name === item.name; }), item);
+    });
+    await Promise.all(changes.map((item) => { return api('PUT', `${path}/${encodeURIComponent(item.name)}`, item); }));
+    return changes.length;
+}
+
+function identity(item) {
+    return { source: item.source, destination: item.destination, kind: item.destination_type || 'queue',
+        key: item.routing_key, arguments: item.arguments || {} };
+}
+
+function bindingPath(path, item) {
+    const kind = item.destination_type === 'exchange' ? 'e' : 'q';
+    return `${path}/e/${encodeURIComponent(item.source)}/${kind}/${encodeURIComponent(item.destination)}`;
+}
+
+async function bindings(api, path, desired, ownsBinding) {
+    const current = await api('GET', path);
+    function includes(rows, item) {
+        return rows.some((row) => { return same(identity(row), identity(item)); });
+    }
+    const missing = desired.filter((item) => { return !includes(current, item); });
+    const stale = current.filter((item) => { return item.source && ownsBinding?.(item) && !includes(desired, item); });
+    await Promise.all(missing.map((item) => {
+        return api('POST', bindingPath(path, item), { routing_key: item.routing_key, arguments: item.arguments || {} });
+    }));
+    await Promise.all(stale.map((item) => {
+        return api('DELETE', `${bindingPath(path, item)}/${encodeURIComponent(item.properties_key)}`);
+    }));
+    return missing.length + stale.length;
+}
+
+async function shovels(api, path, desired, ownsShovel) {
+    const current = await api('GET', path);
+    const changed = desired.filter((item) => {
+        const found = current.find((row) => { return row.name === item.name; });
+        return !same(found, { value: item.value });
+    });
+    const stale = current.filter((item) => {
+        return ownsShovel?.(item) && !desired.some((row) => { return row.name === item.name; });
+    });
+    await Promise.all(changed.map((item) => {
+        return api('PUT', `${path}/${encodeURIComponent(item.name)}`, { value: item.value });
+    }));
+    await Promise.all(stale.map((item) => { return api('DELETE', `${path}/${encodeURIComponent(item.name)}`); }));
+    return changed.length + stale.length;
+}
+
+/** Apply only topology belonging to the selected virtual host. Users and permissions are bootstrap data. */
+export default async function reconcileRabbitmq(desired, options) {
+    const { vhost, ownsBinding, ownsShovel } = options;
+    if (!vhost) { throw new Error('a virtual host is required'); }
+    const api = apiAt(options);
+    const host = encodeURIComponent(vhost);
+    const objects = [...desired.exchanges, ...desired.queues, ...desired.bindings, ...desired.parameters];
+    if (objects.some((item) => { return item.vhost !== undefined && item.vhost !== vhost; })) {
+        throw new Error('topology contains a different virtual host');
+    }
+    if (desired.parameters.some((item) => { return item.component !== 'shovel'; })) {
+        throw new Error('only shovel parameters can be reconciled');
+    }
+    const exchanges = await declare(api, `exchanges/${host}`, desired.exchanges);
+    const queues = await declare(api, `queues/${host}`, desired.queues);
+    const links = await bindings(api, `bindings/${host}`, desired.bindings, ownsBinding);
+    const parameters = await shovels(api, `parameters/shovel/${host}`, desired.parameters, ownsShovel);
+    return exchanges + queues + links + parameters;
+}

package/src/infrastructure/opcua/config.js

@@ -0,0 +1,38 @@
+export function positive(value, name) {
+    if (!Number.isFinite(value) || value <= 0) {
+        throw new TypeError(`${name} must be a positive number`);
+    }
+}
+function validateChannels(channels) {
+    if (!Array.isArray(channels) || channels.length === 0) {
+        throw new TypeError('channels must be a nonempty array');
+    }
+    const nodes = new Set();
+    channels.forEach(({ nodeId }) => {
+        if (typeof nodeId !== 'string' || !nodeId.trim() || nodes.has(nodeId)) {
+            throw new TypeError('invalid or duplicate nodeId');
+        }
+        nodes.add(nodeId);
+    });
+}
+
+function validUrl(value) {
+    const url = new URL(value);
+    return url.protocol === 'opc.tcp:' && Boolean(url.hostname) && !url.username && !url.password;
+}
+export function validateConnections(connections) {
+    if (!Array.isArray(connections)) {
+        throw new TypeError('connections must be an array');
+    }
+    const urls = new Set();
+    connections.forEach((connection) => {
+        if (typeof connection.url !== 'string' || !validUrl(connection.url) || urls.has(connection.url)) {
+            throw new TypeError('invalid or duplicate OPC UA URL');
+        }
+        urls.add(connection.url);
+        positive(connection.publishMs, 'publishMs');
+        positive(connection.heartbeatMs, 'heartbeatMs');
+        validateChannels(connection.channels);
+    });
+    return connections;
+}

package/src/infrastructure/opcua/endpoint.js

@@ -0,0 +1,80 @@
+import { OPCUAClient } from 'node-opcua';
+import session from './session.js';
+
+function report(options, error, connection) {
+    try {
+        options.onError?.(error, connection);
+    } catch {
+        // Observability callbacks must not break endpoint recovery.
+    }
+}
+function delay(state, milliseconds) {
+    return new Promise((resolve) => {
+        state.wake = resolve;
+        state.timer = setTimeout(resolve, milliseconds);
+    }).finally(() => { clearTimeout(state.timer); });
+}
+function clientFor(options) {
+    return OPCUAClient.create({ endpointMustExist: false, keepSessionAlive: false,
+        connectionStrategy: { maxRetry: 0 }, transportTimeout: options.connectTimeoutMs,
+        defaultTransactionTimeout: options.requestTimeoutMs, requestedSessionTimeout: options.requestTimeoutMs * 2 });
+}
+function disconnection(client) {
+    let pending;
+    return () => {
+        pending ??= client.disconnect();
+        return pending;
+    };
+}
+async function attempt(state, connection, publish, options) {
+    const client = clientFor(options);
+    const close = disconnection(client);
+    state.close = close;
+    const current = { active: true, connection, wake: null, timer: null, pending: new Set() };
+    state.current = current;
+    try {
+        await client.connect(connection.url);
+        if (state.active) {
+            await session(current, client, connection, publish, options);
+        }
+    } catch (error) {
+        if (state.active) {
+            report(options, error, connection);
+        }
+    } finally {
+        current.active = false;
+        try { await close(); } finally { await Promise.all(current.pending); }
+    }
+}
+async function run(state, connection, publish, options) {
+    while (state.active) {
+        // Each endpoint deliberately serializes attempts and heartbeat reads.
+        // eslint-disable-next-line no-await-in-loop
+        await attempt(state, connection, publish, options);
+        if (state.active) {
+            // eslint-disable-next-line no-await-in-loop
+            await delay(state, options.retryMs);
+        }
+    }
+}
+export default function endpoint(connection, publish, options) {
+    const state = { active: false, close: null, current: null, task: null, wake: null, timer: null };
+    return {
+        start() {
+            if (!state.active) {
+                state.active = true;
+                state.task = run(state, connection, publish, options);
+            }
+        },
+        async stop() {
+            state.active = false;
+            if (state.current) {
+                state.current.active = false;
+                state.current.wake?.();
+            }
+            state.wake?.();
+            await state.close?.();
+            await state.task;
+        }
+    };
+}

package/src/infrastructure/opcua/index.js

@@ -0,0 +1,10 @@
+import opcuaSource from './source.js';
+export { validateConnections } from './config.js';
+
+/** Compatibility adapter for callback-based SCADA consumers. */
+export default function opcuaConnections(connections, publish, options = {}) {
+    if (typeof publish !== 'function') { throw new TypeError('publish must be a function'); }
+    return opcuaSource(connections, { accept: ({ channel, value }) => { return publish(channel, value); } }, options);
+}
+
+export { default as opcuaSource } from './source.js';