@yarkivaev/scada 2.3.75 → 2.3.77

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.75",
+  "version": "2.3.77",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",

package/src/application/supervisorSink.js

@@ -7,6 +7,7 @@ import migrate, { migratePrivileged } from '../infrastructure/ingest/db/migrate.
 import stateDataFromPool from '../infrastructure/persistence/stateDataFromPool.js';
 import { parseRequestTimeoutMs } from '../infrastructure/ingest/parseRequestTimeoutMs.js';
 import pg from 'pg';
+import { holdClient } from '../infrastructure/persistence/postgresPool.js';
 import path from 'path';
 import { fileURLToPath } from 'url';
 
@@ -59,7 +60,7 @@ export function readDeploymentConfig(env) {
  */
 export default function supervisorSink(env = process.env) {
     const cfg = readDeploymentConfig(env);
-    const pool = new pg.Pool({ connectionString: cfg.postgres });
+    const pool = holdClient(new pg.Pool({ connectionString: cfg.postgres }));
     const segments = segmentPipeline(cfg.stomp, cfg.postgres, pool, cfg.pipeline);
     const decisions = decisionPipeline(cfg.stomp, pool, cfg.pipeline);
     const alerts = alertPipeline(cfg.stomp, pool, cfg.pipeline);
@@ -79,7 +80,7 @@ export default function supervisorSink(env = process.env) {
             if (!cfg.postgresAdmin) {
                 throw new Error('POSTGRES_ADMIN_URL is required for supervisor-sink database migrations');
             }
-            const adminPool = new pg.Pool({ connectionString: cfg.postgresAdmin });
+            const adminPool = holdClient(new pg.Pool({ connectionString: cfg.postgresAdmin }));
             try {
                 await migrate(adminPool, cfg.migrationsDir, cfg.sinkDbProfile);
                 await migratePrivileged(adminPool, cfg.migrationsDir, cfg.sinkDbProfile);

package/src/infrastructure/ingest/durableConsume.js

@@ -0,0 +1,169 @@
+import amqp from 'amqplib';
+
+function connectBroker(url) {
+    return amqp.connect(url, { heartbeat: 10 });
+}
+
+function scheduleLater(fn, pause) {
+    return setTimeout(fn, pause);
+}
+
+function clearLater(handle) {
+    clearTimeout(handle);
+}
+
+function reportFailure(error) {
+    process.stderr.write(`amqp consume failed ${error.message}\n`);
+}
+
+function nextDelay(delay) {
+    return Math.min(delay * 2, 30000);
+}
+
+function blankState() {
+    return {
+        generation: 0,
+        timer: undefined,
+        stopping: false,
+        delay: 1000,
+        session: undefined
+    };
+}
+
+function running(state) {
+    return Boolean(state.session) || (state.generation > 0 && !state.stopping);
+}
+
+function scheduleRetry(ctx, generation) {
+    if (ctx.state.stopping || generation !== ctx.state.generation) {
+        return;
+    }
+    const pause = ctx.state.delay;
+    ctx.state.delay = nextDelay(ctx.state.delay);
+    ctx.state.timer = ctx.schedule(() => {
+        ctx.state.timer = undefined;
+        return ctx.open(generation);
+    }, pause);
+}
+
+function bindClose(ctx, generation, live) {
+    let dropped = false;
+    function lose() {
+        if (dropped || ctx.state.stopping || generation !== ctx.state.generation) {
+            return;
+        }
+        dropped = true;
+        if (ctx.state.session === live) {
+            ctx.state.session = undefined;
+        }
+        scheduleRetry(ctx, generation);
+    }
+    live.connection.on('error', lose);
+    live.connection.on('close', lose);
+    live.channel.on('error', lose);
+    live.channel.on('close', lose);
+}
+
+async function accept(ctx, generation, connection) {
+    if (ctx.state.stopping || generation !== ctx.state.generation) {
+        await connection.close();
+        return null;
+    }
+    return connection.createChannel();
+}
+
+async function keep(ctx, generation, connection, channel, tag) {
+    if (ctx.state.stopping || generation !== ctx.state.generation) {
+        await channel.close();
+        await connection.close();
+        return false;
+    }
+    const live = { connection, channel, tag };
+    ctx.state.session = live;
+    ctx.state.delay = 1000;
+    bindClose(ctx, generation, live);
+    return true;
+}
+
+async function open(ctx, generation) {
+    if (ctx.state.stopping || generation !== ctx.state.generation) {
+        return;
+    }
+    try {
+        const connection = await ctx.connect(ctx.url);
+        const channel = await accept(ctx, generation, connection);
+        if (!channel) {
+            return;
+        }
+        const tag = await ctx.subscribe(channel);
+        await keep(ctx, generation, connection, channel, tag);
+    } catch (error) {
+        ctx.report(error);
+        scheduleRetry(ctx, generation);
+    }
+}
+
+function begin(ctx) {
+    if (running(ctx.state)) {
+        return Promise.resolve();
+    }
+    ctx.state.stopping = false;
+    ctx.state.generation += 1;
+    ctx.state.delay = 1000;
+    return open(ctx, ctx.state.generation);
+}
+
+async function end(ctx) {
+    ctx.state.stopping = true;
+    ctx.state.generation += 1;
+    if (ctx.state.timer) {
+        ctx.clear(ctx.state.timer);
+        ctx.state.timer = undefined;
+    }
+    const live = ctx.state.session;
+    ctx.state.session = undefined;
+    if (!live) {
+        return;
+    }
+    await live.channel.cancel(live.tag);
+    await live.channel.close();
+    await live.connection.close();
+}
+
+/**
+ * Keeps one AMQP consumer subscribed across broker restarts.
+ *
+ * @param {string} url - RabbitMQ AMQP URL
+ * @param {function} subscribe - async (channel) => consumerTag
+ * @param {object} [options] - connect, schedule, clear, report overrides
+ * @returns {object} session with start and stop
+ *
+ * @example
+ *   const session = durableConsume('amqp://localhost', async (channel) => {
+ *       const tag = await channel.consume('scada.telemetry.ingest', () => {});
+ *       return tag.consumerTag;
+ *   });
+ *   await session.start();
+ */
+export default function durableConsume(url, subscribe, options = {}) {
+    const ctx = {
+        url,
+        subscribe,
+        connect: options.connect || connectBroker,
+        schedule: options.schedule || scheduleLater,
+        clear: options.clear || clearLater,
+        report: options.report || reportFailure,
+        state: blankState(),
+        open(generation) {
+            return open(ctx, generation);
+        }
+    };
+    return {
+        start() {
+            return begin(ctx);
+        },
+        stop() {
+            return end(ctx);
+        }
+    };
+}

package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js

@@ -1,5 +1,5 @@
-import amqp from 'amqplib';
 import { batch, circuit, clock, timedBatch } from '@yarkivaev/source-to-sink';
+import durableConsume from '../durableConsume.js';
 import deliverToMqttRecord from './deliverToMqttRecord.js';
 import metricsCodec from '../mqtt/metricsTransformer.js';
 import streamNameFromTopic from './streamNameFromTopic.js';
@@ -50,44 +50,37 @@ export function telemetryConsumer(codec, channel, onSeen) {
  * @param {object} [options] - batch and prefetch options
  * @returns {object} ingest with start and stop
  */
+function subscribeMetrics(queue, prefetch, codec, onSeen) {
+    return async (channel) => {
+        await channel.assertQueue(queue, { durable: true });
+        channel.prefetch(prefetch);
+        const onMessage = telemetryConsumer(codec, channel, onSeen);
+        const tag = await channel.consume(queue, onMessage, { noAck: false });
+        return tag.consumerTag;
+    };
+}
+
 export default function amqpMetricsIngest(amqpUrl, queue, sink, options = {}) {
     const prefetch = options.prefetch || 32;
     const size = options.size || 100;
     const interval = options.interval || 5;
     const threshold = options.threshold || 5;
     const timeout = options.timeout || 60;
-    let session;
     const clk = clock();
     const breaker = circuit(threshold, timeout, clk);
     const collector = timedBatch(batch(sink, size, breaker), interval);
     const codec = metricsCodec(collector);
+    const session = durableConsume(
+        amqpUrl,
+        subscribeMetrics(queue, prefetch, codec, options.onSeen),
+        options
+    );
     return {
-        async start() {
-            if (session) {
-                return;
-            }
-            const conn = await amqp.connect(amqpUrl);
-            const ch = await conn.createChannel();
-            if (session) {
-                await ch.close();
-                await conn.close();
-                return;
-            }
-            await ch.assertQueue(queue, { durable: true });
-            ch.prefetch(prefetch);
-            const onMessage = telemetryConsumer(codec, ch, options.onSeen);
-            const tag = await ch.consume(queue, onMessage, { noAck: false });
-            session = { conn, channel: ch, tag: tag.consumerTag };
+        start() {
+            return session.start();
         },
         async stop() {
-            const active = session;
-            if (!active) {
-                return;
-            }
-            session = undefined;
-            await active.channel.cancel(active.tag);
-            await active.channel.close();
-            await active.conn.close();
+            await session.stop();
             collector.stop();
         }
     };

package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js

@@ -1,4 +1,4 @@
-import amqp from 'amqplib';
+import durableConsume from '../durableConsume.js';
 import deliverToMqttRecord from './deliverToMqttRecord.js';
 
 /**
@@ -34,37 +34,34 @@ export function relayConsumer(sink, channel) {
  * @param {object} [options] - prefetch count
  * @returns {object} Relay with start and stop
  */
+function subscribeRelay(queue, prefetch, sink) {
+    return async (channel) => {
+        await channel.assertQueue(queue, { durable: true });
+        channel.prefetch(prefetch);
+        const onMessage = relayConsumer(sink, channel);
+        const tag = await channel.consume(queue, onMessage, { noAck: false });
+        return tag.consumerTag;
+    };
+}
+
 export default function amqpMqttRelay(amqpUrl, queue, sink, options = {}) {
     const prefetch = options.prefetch || 32;
-    let session;
+    let opened = false;
+    const session = durableConsume(
+        amqpUrl,
+        subscribeRelay(queue, prefetch, sink),
+        options
+    );
     return {
-        async start() {
-            if (session) {
-                return;
+        start() {
+            if (!opened) {
+                opened = true;
+                sink.start();
             }
-            const conn = await amqp.connect(amqpUrl);
-            const ch = await conn.createChannel();
-            if (session) {
-                await ch.close();
-                await conn.close();
-                return;
-            }
-            await ch.assertQueue(queue, { durable: true });
-            ch.prefetch(prefetch);
-            sink.start();
-            const onMessage = relayConsumer(sink, ch);
-            const tag = await ch.consume(queue, onMessage, { noAck: false });
-            session = { conn, channel: ch, tag: tag.consumerTag };
+            return session.start();
         },
         async stop() {
-            const active = session;
-            if (!active) {
-                return;
-            }
-            session = undefined;
-            await active.channel.cancel(active.tag);
-            await active.channel.close();
-            await active.conn.close();
+            await session.stop();
             sink.stop();
         }
     };

package/src/infrastructure/persistence/postgresPool.js

@@ -1,5 +1,24 @@
 import pg from 'pg';
 
+/**
+ * Logs an idle postgres client failure and leaves the pool open.
+ * node-postgres emits `error` when an idle socket dies. With no listener
+ * the process exits.
+ *
+ * @param {import('pg').Pool} pool - long-lived pool
+ * @returns {import('pg').Pool} the same pool
+ *
+ * @example
+ *   holdClient(new pg.Pool({ connectionString: process.env.POSTGRES_URL }));
+ */
+export function holdClient(pool) {
+    const host = pool.options && pool.options.host ? pool.options.host : 'postgres';
+    pool.on('error', (err) => {
+        process.stderr.write(`[scada] postgres idle client failed host=${host} message=${err.message}\n`);
+    });
+    return pool;
+}
+
 /**
  * Creates a PostgreSQL pool for supervisor state reads.
  *
@@ -20,5 +39,5 @@ export default function postgresPool(options = {}) {
     if (!url) {
         return undefined;
     }
-    return new pg.Pool({ connectionString: url });
+    return holdClient(new pg.Pool({ connectionString: url }));
 }

package/src/infrastructure/sync/operationSyncIngest.js

@@ -1,4 +1,4 @@
-import amqp from 'amqplib';
+import durableConsume from '../ingest/durableConsume.js';
 import operationCodec from './operationCodec.js';
 import operationSyncSink from './operationSyncSink.js';
 
@@ -43,40 +43,31 @@ export function operationConsumer(codec, channel) {
  *   const ingest = operationSyncIngest(amqpUrl, 'scada.operations.ingest', dataAccess.operations);
  *   await ingest.start();
  */
+function subscribeOperations(queue, prefetch, codec) {
+    return async (channel) => {
+        await channel.assertQueue(queue, { durable: true });
+        channel.prefetch(prefetch);
+        const onMessage = operationConsumer(codec, channel);
+        const tag = await channel.consume(queue, onMessage, { noAck: false });
+        return tag.consumerTag;
+    };
+}
+
 export default function operationSyncIngest(amqpUrl, queue, operations, options = {}) {
     const prefetch = options.prefetch || 32;
     const sink = operationSyncSink(operations);
     const codec = operationCodec(sink);
-    let session;
+    const session = durableConsume(
+        amqpUrl,
+        subscribeOperations(queue, prefetch, codec),
+        options
+    );
     return {
-        async start() {
-            if (session) {
-                return;
-            }
-            const conn = await amqp.connect(amqpUrl);
-            const ch = await conn.createChannel();
-            await ch.assertQueue(queue, { durable: true });
-            ch.prefetch(prefetch);
-            const onMessage = operationConsumer(codec, ch);
-            const tag = await ch.consume(queue, onMessage, { noAck: false });
-            const started = { conn, channel: ch, tag: tag.consumerTag };
-            if (session) {
-                await ch.cancel(tag.consumerTag);
-                await ch.close();
-                await conn.close();
-                return;
-            }
-            session = started;
+        start() {
+            return session.start();
         },
-        async stop() {
-            const active = session;
-            if (!active) {
-                return;
-            }
-            session = undefined;
-            await active.channel.cancel(active.tag);
-            await active.channel.close();
-            await active.conn.close();
+        stop() {
+            return session.stop();
         }
     };
 }