@yarkivaev/scada 2.3.115 → 2.4.0

package/src/infrastructure/streams/sqliteSink.js

@@ -0,0 +1,55 @@
+import Database from 'better-sqlite3';
+
+/**
+ * SQLite sink for batch record insertion.
+ *
+ * Opens a SQLite database in WAL mode and implements the Sink
+ * interface for use with batch collectors. Creates the target
+ * table with the given columns if it does not exist.
+ *
+ * @example
+ * const sink = sqliteSink('/tmp/metrics.db', 'metrics', ['topic', 'ts', 'value']);
+ * sink.write([{ topic: 'sensor/1', ts: 1700000000000, value: 42 }]);
+ *
+ * @param {string} path - Filesystem path to the SQLite database file
+ * @param {string} table - Target table name
+ * @param {Array<string>} columns - Column names for insertion
+ * @returns {object} Sink with write(records) and close() methods
+ */
+export default function sqliteSink(path, table, columns) {
+  if (typeof path !== 'string' || path.length === 0) {
+    throw new Error('Path must be a non-empty string');
+  }
+  if (typeof table !== 'string' || table.length === 0) {
+    throw new Error('Table must be a non-empty string');
+  }
+  if (!Array.isArray(columns) || columns.length === 0) {
+    throw new Error('Columns must be a non-empty array');
+  }
+  const db = new Database(path);
+  db.pragma('journal_mode = WAL');
+  const placeholders = columns.map(() => {return '?'}).join(', ');
+  const sql = `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${placeholders})`;
+  const statement = db.prepare(sql);
+  const transaction = db.transaction((records) => {
+    for (const record of records) {
+      statement.run(...columns.map((col) => {return record[col]}));
+    }
+  });
+  return {
+    /**
+     * Writes records to SQLite table inside a transaction.
+     *
+     * @param {Array} records - Array of objects with keys matching columns
+     */
+    write(records) {
+      transaction(records);
+    },
+    /**
+     * Closes the database connection.
+     */
+    close() {
+      db.close();
+    }
+  };
+}

package/src/infrastructure/streams/stompBody.js

@@ -0,0 +1,46 @@
+import positive from './positive.js';
+import budgetAccept from './budgetAccept.js';
+
+export function stompSettings(options, prefetch) {
+    const config = { acceptTimeoutMs: 10000, stopTimeoutMs: 5000, maxMessageBytes: 1048576, ...options };
+    config.maxPendingBytes ??= prefetch * config.maxMessageBytes;
+    ['acceptTimeoutMs', 'stopTimeoutMs', 'maxMessageBytes', 'maxPendingBytes'].forEach((key) => { positive(config[key], key); });
+    if (!Number.isSafeInteger(config.maxMessageBytes) || !Number.isSafeInteger(config.maxPendingBytes)) {
+        throw new TypeError('STOMP byte capacities must be safe integers');
+    }
+    return config;
+}
+function overflow() { return new Error('STOMP payload capacity exceeded'); }
+function legacyRead(message, maximum) {
+    return new Promise((resolve, reject) => {
+        message.readString('utf-8', (error, body) => {
+            if (error) { reject(error); }
+            else if (Buffer.byteLength(body) > maximum) { reject(overflow()); }
+            else { resolve(body.toString()); }
+        });
+    });
+}
+/** Native stompit frames are Readables: bound bytes before concatenating the body. */
+export async function readStomp(message, maximum) {
+    if (!message[Symbol.asyncIterator]) { return legacyRead(message, maximum); }
+    const chunks = [];
+    let bytes = 0;
+    for await (const chunk of message) {
+        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        bytes += buffer.length;
+        if (bytes > maximum) { throw overflow(); }
+        chunks.push(buffer);
+    }
+    return Buffer.concat(chunks, bytes).toString('utf8');
+}
+/** Counters and global leases follow real acceptance, including after an uncertain timeout. */
+export async function acceptStomp(ctx, body, settle) {
+    if (!ctx.lifecycle.active) { throw new Error('STOMP source stopped'); }
+    const bytes = Buffer.byteLength(body);
+    if (ctx.lifecycle.bytes + bytes > ctx.lifecycle.maxPendingBytes) { throw overflow(); }
+    ctx.lifecycle.bytes += bytes;
+    try {
+        return await budgetAccept(ctx.collector, { destination: ctx.destination, payload: body, settle }, bytes,
+            ctx.lifecycle, { signal: ctx.lifecycle.controller.signal });
+    } finally { ctx.lifecycle.bytes -= bytes; }
+}

package/src/infrastructure/streams/stompSend.js

@@ -0,0 +1,89 @@
+import stompit from 'stompit';
+import deadline from './deadline.js';
+import positive from './positive.js';
+
+/**
+ * Builds STOMP connect options from a URL.
+ *
+ * @param {URL} parsed - Parsed URL object
+ * @param {object} options - Connection options
+ * @returns {object} stompit connect parameters
+ */
+/**
+ * STOMP SEND headers that persist the body on a durable RabbitMQ queue.
+ *
+ * @param {string} destination - STOMP destination
+ * @returns {object} headers including persistent true
+ */
+export function stompHeaders(destination) {
+    return {
+        destination,
+        'content-type': 'application/json',
+        persistent: 'true'
+    };
+}
+
+function connectParams(parsed, options) {
+    return {
+        host: parsed.hostname,
+        port: parseInt(parsed.port, 10) || 61613,
+        connectHeaders: {
+            host: options.host || parsed.hostname,
+            login: options.login || '',
+            passcode: options.passcode || ''
+        }
+    };
+}
+
+/**
+ * One-shot STOMP publish of a JSON payload to a destination.
+ *
+ * Connects, sends one MESSAGE frame, disconnects. For operator decisions
+ * and other low-volume outbound events.
+ *
+ * @example
+ * await stompSend('stomp://rabbitmq:61613', '/exchange/scada.user_decisions',
+ *   { machine: 'icht2', start: 1700000000, user: 'hmi', tags: ['heating'] },
+ *   { login: 'guest', passcode: 'guest', host: '/' });
+ *
+ * @param {string} url - STOMP broker URL
+ * @param {string} destination - STOMP destination
+ * @param {object|string} payload - JSON-serializable body or raw string
+ * @param {object} [options] - STOMP connection options
+ * @param {string} [options.login] - Login
+ * @param {string} [options.passcode] - Passcode
+ * @param {string} [options.host] - Virtual host header
+ * @returns {Promise<void>} Resolves when the frame is sent and client disconnects
+ */
+export default function stompSend(url, destination, payload, options = {}) {
+    if (typeof url !== 'string' || url.length === 0) {
+        throw new Error('URL must be a non-empty string');
+    }
+    if (typeof destination !== 'string' || destination.length === 0) {
+        throw new Error('Destination must be a non-empty string');
+    }
+    const parsed = new URL(url);
+    const params = connectParams(parsed, options);
+    const body = typeof payload === 'string' ? payload : JSON.stringify(payload);
+    const timeoutMs = options.timeoutMs ?? 10000;
+    positive(timeoutMs, 'timeoutMs');
+    const state = { active: true, client: null, connecting: null };
+    const sent = new Promise((resolve, reject) => {
+        state.connecting = (options.connect || stompit.connect)({ ...params, timeout: timeoutMs }, (error, client) => {
+            if (error) { reject(error); return; }
+            state.client = client;
+            if (!state.active) { client.destroy(); return; }
+            client.on('error', reject);
+            try {
+                const frame = client.send(stompHeaders(destination), { onReceipt() {
+                    if (state.active) { client.disconnect((failure) => { if (failure) { reject(failure); } else { resolve(); } }); }
+                } });
+                frame.end(body);
+            } catch (failure) { client.destroy(); reject(failure); }
+        });
+    });
+    return deadline(sent, timeoutMs, 'STOMP receipt/disconnect', () => {
+        state.active = false;
+        (state.client || state.connecting)?.destroy();
+    }).finally(() => { state.active = false; });
+}

package/src/infrastructure/streams/stompSource.js

@@ -0,0 +1,201 @@
+import stompit from 'stompit';
+import follow from './follow.js';
+import deadline from './deadline.js';
+import { acceptStomp, readStomp, stompSettings } from './stompBody.js';
+
+/**
+ * Idle state for STOMP source.
+ *
+ * @returns {object} State with subscribed() returning false
+ */
+function idle() {
+  return {
+    subscribed() {
+      return false;
+    }
+  };
+}
+
+/**
+ * Subscribed state for STOMP source.
+ *
+ * @param {object} channel - STOMP channel
+ * @returns {object} State with subscribed() returning true
+ */
+function subscribed(channel, delivery) {
+  return {
+    subscribed() {
+      return true;
+    },
+    channel() {
+      return channel;
+    },
+    delivery
+  };
+}
+
+/**
+ * Builds STOMP server configuration from a URL.
+ *
+ * @param {URL} parsed - Parsed URL object
+ * @param {object} options - Connection options
+ * @returns {object} Server config for ConnectFailover
+ */
+function serverConfig(parsed, options) {
+  return {
+    host: parsed.hostname,
+    port: parseInt(parsed.port, 10) || 61613,
+    connectHeaders: {
+      host: options.host || parsed.hostname,
+      login: options.login || '',
+      passcode: options.passcode || ''
+    }
+  };
+}
+
+/**
+ * Builds settle helpers for one STOMP frame.
+ *
+ * @param {object} channel - STOMP channel
+ * @param {object} message - STOMP frame
+ * @returns {{ ack: Function, nack: Function }}
+ */
+function settleOf(channel, message, lifecycle) {
+  let settled = false;
+  return {
+    ack() {
+      if (lifecycle.active && !settled) { channel.ack(message); settled = true; }
+    },
+    nack(requeue = true) {
+      if (lifecycle.active && !settled) { channel.nack(message, { requeue: String(requeue) }); settled = true; }
+    }
+  };
+}
+
+/**
+ * Delivers one STOMP body to the collector and settles the frame.
+ *
+ * @param {object} ctx - delivery context with channel/message/destination/collector/manualAck
+ * @param {string} body - UTF-8 payload
+ * @returns {Promise<void>}
+ */
+async function deliver(ctx, body) {
+  const settle = settleOf(ctx.channel, ctx.message, ctx.lifecycle);
+  try {
+    await deadline(Promise.resolve().then(() => {
+      return acceptStomp(ctx, body, settle);
+    }), ctx.lifecycle.acceptTimeoutMs, 'STOMP acceptance', (error) => {
+      ctx.lifecycle.fatal = error; ctx.lifecycle.active = false; ctx.lifecycle.controller.abort(error); ctx.channel.close();
+    });
+    if (!ctx.manualAck) {
+      settle.ack();
+    }
+  } catch (error) {
+    if (!ctx.manualAck) {
+      settle.nack();
+    }
+    throw error;
+  }
+}
+
+/**
+ * Subscribes a channel and routes frames to the collector.
+ * Arrivals stay in subscribe order through follow, even when persist overlaps.
+ *
+ * @param {object} channel - STOMP channel
+ * @param {string} destination - queue destination
+ * @param {object} collector - message collector
+ * @param {object} delivery - serial, manualAck and prefetch settings
+ * @returns {undefined}
+ */
+function bind(channel, destination, collector, delivery) {
+  const { serial, manualAck, prefetch, lifecycle } = delivery;
+  let pending = Promise.resolve();
+  channel.subscribe({ destination, ack: 'client-individual', 'prefetch-count': String(prefetch) }, (err, message) => {
+    if (err) {
+      return;
+    }
+    const ctx = { channel, message, destination, collector, manualAck, lifecycle };
+    pending = follow(pending, () => {
+      if (!lifecycle.active) { return Promise.reject(new Error('STOMP source stopped')); }
+      return deadline(readStomp(message, lifecycle.maxMessageBytes), lifecycle.acceptTimeoutMs, 'STOMP read').catch((error) => {
+        lifecycle.fatal = error; lifecycle.active = false; channel.close(); throw error;
+      });
+    }, (body) => {
+      const task = deliver(ctx, body);
+      lifecycle.pending.add(task);
+      return task.finally(() => { lifecycle.pending.delete(task); });
+    }, serial);
+    lifecycle.chain = pending;
+    void pending.catch((error) => {
+      try { lifecycle.onError?.(error); } catch {
+        // Reporting cannot reject a transport callback.
+      }
+    });
+  });
+}
+
+/**
+ * STOMP subscription source for streaming messages to a collector.
+ *
+ * Messages are `{destination, payload, settle}`. Default ack-after-accept;
+ * with `manualAck` the collector owns settle. Arrivals enqueue in
+ * subscribe order. With `serial: false` persist may overlap after enqueue.
+ * A finite prefetch (default 32) bounds unacknowledged broker deliveries.
+ *
+ * @param {string} url - STOMP broker URL
+ * @param {string} destination - STOMP destination
+ * @param {object} collector - Collector with accept()
+ * @param {object} [options] - Connection and delivery options
+ * @param {number} [options.prefetch=32] - Positive integer delivery window
+ * @returns {object} Source with start() and stop() methods
+ */
+export default function stompSource(url, destination, collector, options = {}) {
+  if (typeof url !== 'string' || url.length === 0) {
+    throw new Error('URL must be a non-empty string');
+  }
+  if (typeof destination !== 'string' || destination.length === 0) {
+    throw new Error('Destination must be a non-empty string');
+  }
+  if (!collector || typeof collector.accept !== 'function') {
+    throw new Error('Collector must have an accept() method');
+  }
+  const prefetch = options.prefetch ?? 32;
+  if (!Number.isSafeInteger(prefetch) || prefetch < 1) {
+    throw new Error('prefetch must be a positive safe integer');
+  }
+  const settings = stompSettings(options, prefetch);
+  const parsed = new URL(url);
+  const serial = options.serial !== false;
+  const manualAck = options.manualAck === true;
+  let state = idle();
+  return {
+    start() {
+      if (state.subscribed()) {
+        return;
+      }
+      const failover = new stompit.ConnectFailover([serverConfig(parsed, options)], {
+        initialReconnectDelay: 10,
+        maxReconnectDelay: 30000,
+        maxReconnects: -1
+      });
+      const channel = new stompit.Channel(failover);
+      const lifecycle = { ...settings, active: true, controller: new AbortController(), pending: new Set(), bytes: 0 };
+      bind(channel, destination, collector, { serial, manualAck, prefetch, lifecycle });
+      state = subscribed(channel, lifecycle);
+    },
+    stop() {
+      if (!state.subscribed()) {
+        return Promise.resolve();
+      }
+      const stopped = state;
+      stopped.delivery.active = false;
+      stopped.delivery.controller.abort(new Error('STOMP source stopped'));
+      stopped.channel().close();
+      return deadline(Promise.allSettled([stopped.delivery.chain, ...stopped.delivery.pending]), options.stopTimeoutMs ?? 5000, 'STOMP stop').then(() => {
+        if (stopped.delivery.fatal) { throw stopped.delivery.fatal; }
+        if (state === stopped) { state = idle(); }
+      });
+    }
+  };
+}

package/src/infrastructure/sync/operationSyncIngest.js

@@ -1,4 +1,4 @@
-import durableConsume from '../ingest/durableConsume.js';
+import { amqpSource } from '../streams/index.js';
 import operationCodec from './operationCodec.js';
 import operationSyncSink from './operationSyncSink.js';
 
@@ -43,25 +43,13 @@ export function operationConsumer(codec, channel) {
  *   const ingest = operationSyncIngest(amqpUrl, 'scada.operations.ingest', dataAccess.operations);
  *   await ingest.start();
  */
-function subscribeOperations(queue, prefetch, codec) {
-    return async (channel) => {
-        await channel.assertQueue(queue, { durable: true });
-        channel.prefetch(prefetch);
-        const onMessage = operationConsumer(codec, channel);
-        const tag = await channel.consume(queue, onMessage, { noAck: false });
-        return tag.consumerTag;
-    };
-}
-
 export default function operationSyncIngest(amqpUrl, queue, operations, options = {}) {
     const prefetch = options.prefetch || 32;
     const sink = operationSyncSink(operations);
     const codec = operationCodec(sink);
-    const session = durableConsume(
-        amqpUrl,
-        subscribeOperations(queue, prefetch, codec),
-        options
-    );
+    const session = amqpSource(amqpUrl, queue, {
+        accept(delivery) { return acceptOperationDeliver(codec, delivery.content); }
+    }, { ...options, prefetch });
     return {
         start() {
             return session.start();