@yarkivaev/scada 2.4.4 → 2.4.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.4.4",
+  "version": "2.4.5",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",

package/src/application/ingestHealth.js

@@ -0,0 +1,17 @@
+/** Process health follows all owned readers, not merely the HTTP listener. */
+export default function ingestHealth({ onFatal } = {}) {
+    const state = { sources: [], started: false, fatal: null };
+    return {
+        track(sources) { state.sources = sources; },
+        started() { state.started = true; },
+        fail(error) {
+            if (state.fatal) { return; }
+            state.fatal = error;
+            onFatal?.(error);
+        },
+        status() {
+            return { live: !state.fatal, ready: state.started && !state.fatal
+                && state.sources.every((source) => {return source.health().state === 'running'}) };
+        }
+    };
+}

package/src/application/runSupervisorSink.js

@@ -0,0 +1,16 @@
+import supervisorSink from './supervisorSink.js';
+import runRetention from '../infrastructure/ingest/db/runRetention.js';
+import edgeApi from '../infrastructure/http/edge/edgeApi.js';
+
+/** A terminal reader error rejects the process lifetime, not only one detached task. */
+export default async function runSupervisorSink(env, { sinkFactory = supervisorSink, apiFactory = edgeApi } = {}) {
+    let fail;
+    const failed = new Promise((_resolve, reject) => { fail = reject; });
+    const sink = sinkFactory(env, { onFatal: fail });
+    const api = apiFactory(sink.dataAccess, {
+        port: sink.apiPort, token: sink.apiToken, metricsEnabled: sink.metricsEnabled,
+        retentionEnabled: sink.retentionEnabled, retentionDays: sink.retentionDays,
+        requestTimeoutMs: sink.requestTimeoutMs, pool: sink.pool, runRetention, health: sink.health
+    });
+    await Promise.all([sink.run(api), failed]);
+}

package/src/application/runtime/additionalAdapters.js

@@ -1,3 +1,4 @@
+import ingestHealth from '../ingestHealth.js';
 import { clickhouseSink } from '../../infrastructure/streams/index.js';
 import amqpMetricsIngest from '../../infrastructure/ingest/telemetry/amqpMetricsIngest.js';
 import operationSyncIngest from '../../infrastructure/sync/operationSyncIngest.js';
@@ -70,12 +71,23 @@ function silent(spec, pool, context) {
 }
 function stomp(spec, pool, context) {
     const shared = borrowed(pool);
-    const config = { ...spec.config, ...boundedOptions({ ...spec, options: spec.config }), budget: context.budget };
-    return together([
-        segmentPipeline(spec.stomp, spec.postgres, shared, config),
-        decisionPipeline(spec.stomp, shared, config),
-        alertPipeline(spec.stomp, shared, config)
-    ]);
+    const health = ingestHealth({ onFatal(error) {
+        context.report?.({ state: 'failed', fatal: true, reason: error.code ?? error.name });
+    } });
+    function report() {
+        const status = health.status();
+        let state = status.ready ? 'running' : 'degraded';
+        if (!status.live) { state = 'failed'; }
+        context.report?.({ state, fatal: !status.live });
+    }
+    const config = { ...spec.config, ...boundedOptions({ ...spec, options: spec.config }), budget: context.budget,
+        onFatal(error) { health.fail(error); }, onState: report };
+    const items = [segmentPipeline(spec.stomp, spec.postgres, shared, config),
+        decisionPipeline(spec.stomp, shared, config), alertPipeline(spec.stomp, shared, config)];
+    health.track(items);
+    const group = together(items);
+    return { async start() { await group.start(); health.started(); report(); },
+        stop() { return group.stop(); } };
 }
 function close(state) {
     state.closing ??= Promise.resolve().then(async () => {

package/src/application/runtime/taskContext.js

@@ -9,6 +9,7 @@ export default function taskContext(slot, request, ctx) {
         report(event) {
             if (controller.signal.aborted) { return; }
             slot.health = event.state;
+            if (event.fatal === true) { ctx.fatal = true; }
             if (event.delivered) { slot.lastDelivery = Date.now(); }
             try { ctx.services.report?.({ taskId: slot.taskId, generation: slot.generation, ...event }); } catch {
                 // Observability cannot interrupt collection.

package/src/application/supervisorSink.js

@@ -1,3 +1,4 @@
+import ingestHealth from './ingestHealth.js';
 import segmentPipeline, {
     segmentsIngestDestination
 } from '../infrastructure/ingest/pipelines/segmentPipeline.js';
@@ -21,6 +22,12 @@ const appRoot = path.dirname(fileURLToPath(import.meta.url));
  * @param {NodeJS.ProcessEnv} env - process environment
  * @returns {object} deployment configuration
  */
+function milliseconds(env, key, fallback) {
+    const value = Number(env[key] ?? fallback);
+    if (!Number.isSafeInteger(value) || value < 1) { throw new TypeError(`${key} must be a positive integer`); }
+    return value;
+}
+
 export function readDeploymentConfig(env) {
     const root = path.join(appRoot, '../..');
     const prefetch = Number(env.STOMP_PREFETCH || '128');
@@ -44,6 +51,8 @@ export function readDeploymentConfig(env) {
         pipelines: env.COLLECTOR_MODE !== 'external' && env.SINK_PIPELINES !== 'false',
         pipeline: {
             prefetch,
+            acceptTimeoutMs: milliseconds(env, 'STOMP_ACCEPT_TIMEOUT_MS', 10000),
+            stopTimeoutMs: milliseconds(env, 'STOMP_STOP_TIMEOUT_MS', 5000),
             deadLetter: env.SEGMENTS_DEAD_LETTER_ENABLED === 'true',
             size: parseInt(env.BATCH_SIZE || '50', 10),
             interval: parseInt(env.FLUSH_INTERVAL || '2', 10),
@@ -60,9 +69,10 @@ export function readDeploymentConfig(env) {
     };
 }
 
-function startPipelines(cfg, pool) {
+function startPipelines(cfg, pool, health) {
     const pipelines = [segmentPipeline(cfg.stomp, cfg.postgres, pool, cfg.pipeline),
         decisionPipeline(cfg.stomp, pool, cfg.pipeline), alertPipeline(cfg.stomp, pool, cfg.pipeline)];
+    health.track(pipelines);
     return Promise.all(pipelines.map((pipeline) => { return pipeline.start(); }));
 }
 
@@ -73,12 +83,15 @@ function startPipelines(cfg, pool) {
  * @param {NodeJS.ProcessEnv} [env] - process environment, defaults to process.env
  * @returns {object} sink runtime with pool, dataAccess, and run(httpServer?)
  */
-export default function supervisorSink(env = process.env) {
+export default function supervisorSink(env = process.env, options = {}) {
     const cfg = readDeploymentConfig(env);
+    const health = ingestHealth(options);
+    cfg.pipeline.onFatal = (error) => { health.fail(error); };
     const pool = holdClient(new pg.Pool({ connectionString: cfg.postgres }));
     const metricsEnabled = cfg.sinkDbProfile === 'edge';
     const dataAccess = stateDataFromPool(pool, { metricsEnabled });
     return {
+        health: () => {return health.status()},
         apiPort: cfg.apiPort,
         apiToken: cfg.apiToken,
         sinkDbProfile: cfg.sinkDbProfile,
@@ -100,8 +113,9 @@ export default function supervisorSink(env = process.env) {
                 await adminPool.end();
             }
             if (cfg.pipelines) {
-                await startPipelines(cfg, pool);
+                await startPipelines(cfg, pool, health);
             }
+            health.started();
             if (httpServer) {
                 httpServer.start();
             }

package/src/bin/supervisor-sink.js

@@ -1,28 +1,12 @@
-/**
- * Supervisor-sink service entry point.
- */
-process.on('unhandledRejection', (err) => {
-    process.stderr.write(`Unhandled rejection: ${String(err)}\n`);
-    process.exit(1);
-});
+/** Supervisor-sink process; a fenced reader must not leave a healthy zombie HTTP server. */
+import runSupervisorSink from '../application/runSupervisorSink.js';
 
-import supervisorSink from '../application/supervisorSink.js';
-import runRetention from '../infrastructure/ingest/db/runRetention.js';
-import edgeApi from '../infrastructure/http/edge/edgeApi.js';
-
-const sink = supervisorSink(process.env);
-const http = edgeApi(sink.dataAccess, {
-    port: sink.apiPort,
-    token: sink.apiToken,
-    metricsEnabled: sink.metricsEnabled,
-    retentionEnabled: sink.retentionEnabled,
-    retentionDays: sink.retentionDays,
-    requestTimeoutMs: sink.requestTimeoutMs,
-    pool: sink.pool,
-    runRetention
+process.on('unhandledRejection', (error) => {
+    process.stderr.write(`Unhandled rejection: ${String(error)}\n`);
+    process.exit(1);
 });
 
-sink.run(http).catch((error) => {
-    process.stderr.write(`${String(error)}\n`);
+runSupervisorSink(process.env).catch((error) => {
+    process.stderr.write(`Supervisor sink failed: ${String(error)}\n`);
     process.exit(1);
 });

package/src/infrastructure/http/edge/edgeApi.js

@@ -1,4 +1,5 @@
 import http from 'http';
+import ingestHealthRoutes from './routes/ingestHealthRoutes.js';
 import checkpointRoutes from './routes/checkpoint/checkpointRoutes.js';
 import metricsRoutes from './routes/metrics/metricsRoutes.js';
 import retentionRoutes from './routes/retention/retentionRoutes.js';
@@ -25,6 +26,7 @@ export function createEdgeApi(dataAccess, options) {
         ? metricsRoutes(token, metrics)
         : [];
     return routes([
+        ...(opt.health ? ingestHealthRoutes(opt.health) : []),
         ...admin,
         ...checkpointRoutes(token, checkpoints),
         ...metricRoutes

package/src/infrastructure/http/edge/routes/ingestHealthRoutes.js

@@ -0,0 +1,10 @@
+import { route } from '@yarkivaev/simple-server';
+
+/** Health never exposes credentials, messages or application records. */
+export default function ingestHealthRoutes(health) {
+    return [['/ready', 'ready'], ['/live', 'live']].map(([path, key]) => {return route('GET', path, (_req, res) => {
+        const available = health()[key] === true;
+        res.writeHead(available ? 200 : 503, { 'content-type': 'application/json', 'cache-control': 'no-store' });
+        res.end(JSON.stringify({ [key]: available }));
+    })});
+}

package/src/infrastructure/ingest/pipelines/alertPipeline.js

@@ -32,6 +32,7 @@ export default function alertPipeline(stomp, pool, config) {
     const source = stompSource(stomp, '/exchange/scada.alerts', codec,
         stompOptions(config));
     return {
+        health() { return source.health(); },
         /**
          * Starts the pipeline.
          */

package/src/infrastructure/ingest/pipelines/decisionPipeline.js

@@ -29,6 +29,7 @@ export default function decisionPipeline(stomp, pool, config) {
     const source = stompSource(stomp, '/exchange/scada.user_decisions', codec,
         stompOptions(config));
     return {
+        health() { return source.health(); },
         /**
          * Starts consuming user decisions from RabbitMQ.
          */

package/src/infrastructure/ingest/pipelines/segmentPipeline.js

@@ -1,4 +1,5 @@
 import stompOptions from './stompOptions.js';
+import deadline from '../../streams/deadline.js';
 import { postgresSink, stompSource } from '../../streams/index.js';
 import segmentDispatch from '../../../domain/segment/dispatch.js';
 import segmentCoalesce from '../../../domain/segment/segmentCoalesce.js';
@@ -42,8 +43,8 @@ function segmentSinks(postgres, pool) {
 /**
  * Pipeline for streaming STOMP segment data to PostgreSQL segments table.
  * Subscribes to the durable ingest queue so shovel traffic survives consumer gaps.
- * Pending heartbeats are coalesced per machine; machines persist in
- * parallel lanes. Arrivals enqueue in broker FIFO before coalesce.
+ * Pending heartbeats are coalesced per machine. Source acceptance is FIFO:
+ * queued lane work must not consume another delivery's execution deadline.
  *
  * @param {string} stomp - STOMP broker URL
  * @param {string} postgres - PostgreSQL connection URL
@@ -56,17 +57,26 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
     const { segmentSink, splitSink } = segmentSinks(postgres, pool);
     const continued = continuedSegments(segmentSink, pool, budget);
     const dispatch = segmentDispatch(continued, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
-    const coalesce = segmentCoalesce(segmentCodec(dispatch), {
+    // eslint-disable-next-line prefer-const -- The timer codec closes over the source constructed with that codec.
+    let source;
+    const codec = segmentCodec(dispatch);
+    const bounded = { accept(envelope) {
+        return deadline(Promise.resolve().then(() => { return codec.accept(envelope); }),
+            config.acceptTimeoutMs ?? 10000, 'STOMP acceptance', (error) => { source.fail(error); });
+    } };
+    // Heartbeat timer flushes outlive source.accept and need their own execution deadline.
+    const coalesce = segmentCoalesce(bounded, {
         size: config.size,
         deadLetter: config.deadLetter,
         interval: config.interval
     });
     const destination = config.segmentsDestination || segmentsIngestDestination;
-    const source = stompSource(stomp, destination, coalesce, stompOptions(config, { serial: false, manualAck: true }));
+    source = stompSource(stomp, destination, coalesce, stompOptions(config, { serial: true, manualAck: true }));
     const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
         intervalMs: (config.poll || 5) * 1000
     });
     return {
+        health() { return source.health(); },
         destination,
         async start() {
             const repair = abutSegments(pool, budget);

package/src/infrastructure/ingest/pipelines/stompOptions.js

@@ -2,6 +2,8 @@
 export default function stompOptions(config, overrides = {}) {
     const options = { login: config.login, passcode: config.passcode, host: config.host,
         prefetch: config.prefetch, maxMessageBytes: config.maxMessageBytes,
-        maxPendingBytes: config.maxPendingBytes, budget: config.budget, ...overrides };
+        maxPendingBytes: config.maxPendingBytes, budget: config.budget,
+        acceptTimeoutMs: config.acceptTimeoutMs, stopTimeoutMs: config.stopTimeoutMs,
+        onError: config.onError, onFatal: config.onFatal, onState: config.onState, ...overrides };
     return Object.fromEntries(Object.entries(options).filter(([, value]) => { return value !== undefined; }));
 }

package/src/infrastructure/streams/follow.js

@@ -26,7 +26,7 @@ function resume(pending) {
  *   let pending = Promise.resolve();
  *   pending = follow(pending, () => Promise.resolve('a'), write, false);
  */
-export default function follow(pending, read, deliver, serial) {
+export default function follow(pending, read, deliver, serial, onError = (error) => { process.stderr.write(`Delivery failed: ${error.message}\n`); }) {
     return resume(pending).then(async () => {
         const body = await read();
         const run = deliver(body);
@@ -34,7 +34,9 @@ export default function follow(pending, read, deliver, serial) {
             return run;
         }
         void run.catch((error) => {
-            return error;
+            try { onError(error); } catch {
+                // Reporting cannot turn a detached delivery into an unhandled rejection.
+            }
         });
         return undefined;
     });

package/src/infrastructure/streams/stompLifecycle.js

@@ -0,0 +1,46 @@
+/** A fenced subscription reports failure once and cannot be revived by late callbacks. */
+export default function stompLifecycle(settings, channel, destination) {
+    const state = { ...settings, active: true, fatal: null, phase: 'starting', connected: false,
+        controller: new AbortController(), pending: new Set(), bytes: 0, closed: false };
+    function notify(callback, value) {
+        try { callback?.(value); } catch {
+            // An observer cannot interrupt transport fencing or broker callbacks.
+        }
+    }
+    function close() {
+        if (!state.closed) { state.closed = true; channel.close(); }
+    }
+    function report(error) {
+        if (settings.onError) { notify(settings.onError, error); }
+        else { process.stderr.write(`STOMP source ${destination}: ${error.message}\n`); }
+    }
+    return Object.assign(state, {
+        transition(phase) {
+            if (!state.active || state.fatal) { return; }
+            state.phase = phase;
+            if (phase === 'running') { state.connected = true; }
+            notify(settings.onState, { state: phase });
+        },
+        fail(error) {
+            if (!state.active || state.fatal) { return; }
+            state.fatal = error;
+            state.active = false;
+            state.phase = 'failed';
+            state.controller.abort(error);
+            close();
+            report(error);
+            notify(settings.onState, { state: 'failed', reason: error.message });
+            notify(settings.onFatal, error);
+        },
+        report(error) {
+            if (state.active && error !== state.fatal) { report(error); }
+        },
+        halt() {
+            state.active = false;
+            state.controller.abort(new Error('STOMP source stopped'));
+            if (!state.fatal) { state.phase = 'stopped'; }
+            close();
+        },
+        health() { return { state: state.phase, reason: state.fatal?.message ?? null }; }
+    });
+}

package/src/infrastructure/streams/stompSource.js

@@ -1,5 +1,6 @@
 import stompit from 'stompit';
 import follow from './follow.js';
+import stompLifecycle from './stompLifecycle.js';
 import deadline from './deadline.js';
 import { acceptStomp, readStomp, stompSettings } from './stompBody.js';
 
@@ -85,7 +86,7 @@ async function deliver(ctx, body) {
     await deadline(Promise.resolve().then(() => {
       return acceptStomp(ctx, body, settle);
     }), ctx.lifecycle.acceptTimeoutMs, 'STOMP acceptance', (error) => {
-      ctx.lifecycle.fatal = error; ctx.lifecycle.active = false; ctx.lifecycle.controller.abort(error); ctx.channel.close();
+      ctx.lifecycle.fail(error);
     });
     if (!ctx.manualAck) {
       settle.ack();
@@ -113,25 +114,22 @@ function bind(channel, destination, collector, delivery) {
   let pending = Promise.resolve();
   channel.subscribe({ destination, ack: 'client-individual', 'prefetch-count': String(prefetch) }, (err, message) => {
     if (err) {
+      lifecycle.fail(err);
       return;
     }
     const ctx = { channel, message, destination, collector, manualAck, lifecycle };
     pending = follow(pending, () => {
       if (!lifecycle.active) { return Promise.reject(new Error('STOMP source stopped')); }
       return deadline(readStomp(message, lifecycle.maxMessageBytes), lifecycle.acceptTimeoutMs, 'STOMP read').catch((error) => {
-        lifecycle.fatal = error; lifecycle.active = false; channel.close(); throw error;
+        lifecycle.fail(error); throw error;
       });
     }, (body) => {
       const task = deliver(ctx, body);
       lifecycle.pending.add(task);
       return task.finally(() => { lifecycle.pending.delete(task); });
-    }, serial);
+    }, serial, (error) => { lifecycle.report(error); });
     lifecycle.chain = pending;
-    void pending.catch((error) => {
-      try { lifecycle.onError?.(error); } catch {
-        // Reporting cannot reject a transport callback.
-      }
-    });
+    void pending.catch((error) => { lifecycle.report(error); });
   });
 }
 
@@ -150,6 +148,12 @@ function bind(channel, destination, collector, delivery) {
  * @param {number} [options.prefetch=32] - Positive integer delivery window
  * @returns {object} Source with start() and stop() methods
  */
+function watchConnection(failover, lifecycle) {
+  failover.on('connect', () => { lifecycle.transition('running'); });
+  failover.on('connecting', () => { lifecycle.transition(lifecycle.connected ? 'degraded' : 'starting'); });
+  failover.on('error', (error) => { lifecycle.transition('degraded'); lifecycle.report(error); });
+}
+
 export default function stompSource(url, destination, collector, options = {}) {
   if (typeof url !== 'string' || url.length === 0) {
     throw new Error('URL must be a non-empty string');
@@ -172,6 +176,7 @@ export default function stompSource(url, destination, collector, options = {}) {
   return {
     start() {
       if (state.subscribed()) {
+        if (state.delivery.fatal) { throw state.delivery.fatal; }
         return;
       }
       const failover = new stompit.ConnectFailover([serverConfig(parsed, options)], {
@@ -180,18 +185,18 @@ export default function stompSource(url, destination, collector, options = {}) {
         maxReconnects: -1
       });
       const channel = new stompit.Channel(failover);
-      const lifecycle = { ...settings, active: true, controller: new AbortController(), pending: new Set(), bytes: 0 };
+      const lifecycle = stompLifecycle(settings, channel, destination);
+      watchConnection(failover, lifecycle);
       bind(channel, destination, collector, { serial, manualAck, prefetch, lifecycle });
       state = subscribed(channel, lifecycle);
     },
+    // Asynchronous downstream work uses the same fence as delivery deadlines.
+    fail(error) { if (state.subscribed()) { state.delivery.fail(error); } },
+    health() { return state.subscribed() ? state.delivery.health() : { state: 'idle', reason: null }; },
     stop() {
-      if (!state.subscribed()) {
-        return Promise.resolve();
-      }
+      if (!state.subscribed()) { return Promise.resolve(); }
       const stopped = state;
-      stopped.delivery.active = false;
-      stopped.delivery.controller.abort(new Error('STOMP source stopped'));
-      stopped.channel().close();
+      stopped.delivery.halt();
       return deadline(Promise.allSettled([stopped.delivery.chain, ...stopped.delivery.pending]), options.stopTimeoutMs ?? 5000, 'STOMP stop').then(() => {
         if (stopped.delivery.fatal) { throw stopped.delivery.fatal; }
         if (state === stopped) { state = idle(); }