@yarkivaev/scada 2.3.56 → 2.3.58

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.56",
+  "version": "2.3.58",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",
@@ -29,7 +29,7 @@
   "dependencies": {
     "@clickhouse/client": "^1.0.0",
     "@yarkivaev/simple-server": "1.0.1",
-    "@yarkivaev/source-to-sink": "1.3.2",
+    "@yarkivaev/source-to-sink": "1.3.4",
     "amqplib": "^0.10.9",
     "pg": "^8.0.0",
     "stompit": "^1.0.0"

package/src/domain/segment/dispatch.js

@@ -1,31 +1,69 @@
+/**
+ * Persists a pending heartbeat, optionally inside one pool transaction.
+ *
+ * @param {object} segmentSink - upsert sink
+ * @param {object} closer - orphan closer
+ * @param {object} [pool] - optional pg pool
+ * @param {object} record - normalized segment record
+ * @returns {Promise<void>}
+ */
+async function persistPending(segmentSink, closer, pool, record) {
+    const kind = record.kind || 'phase';
+    if (!pool || typeof pool.connect !== 'function') {
+        await closer.close(record.machine, record.start_time, kind);
+        await segmentSink.write([record]);
+        return;
+    }
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        await closer.close(record.machine, record.start_time, kind, client);
+        await segmentSink.write([record], client);
+        await client.query('COMMIT');
+    } catch (error) {
+        await client.query('ROLLBACK').catch((ignored) => { return ignored; });
+        throw error;
+    } finally {
+        client.release();
+    }
+}
+
 /**
  * Routes normalized segment records to the correct persistence sink.
  *
+ * When a pool is supplied, pending segment heartbeats (`duration === 0`)
+ * run orphan-close and upsert inside one transaction on a single client.
+ *
  * @param {object} segmentSink - batch insert/update sink
  * @param {object} retag - retag sink
  * @param {object} splitSink - split update sink
  * @param {object} closer - orphan open segment closer
+ * @param {object} [pool] - optional pg pool for transactional pending writes
  * @returns {object} collector with accept(record)
  *
  * @example
- *   const route = segmentDispatch(segmentSink, retag, splitSink, closer);
+ *   const route = segmentDispatch(segmentSink, retag, splitSink, closer, pool);
  *   await route.accept({ type: 'segment', machine: 'm1', start_time: '...', duration: 0 });
  */
-export default function segmentDispatch(segmentSink, retag, splitSink, closer) {
+export default function segmentDispatch(segmentSink, retag, splitSink, closer, pool) {
     return {
         async accept(record) {
             if (record.type === 'retag') {
                 await retag.accept(record);
-            } else if (record.type === 'split') {
+                return;
+            }
+            if (record.type === 'split') {
                 await splitSink.write([record]);
-            } else if (record.type === 'segment') {
-                if (record.duration === 0) {
-                    await closer.close(record.machine, record.start_time, record.kind || 'phase');
-                }
-                await segmentSink.write([record]);
-            } else {
+                return;
+            }
+            if (record.type !== 'segment') {
                 throw new Error(`Segment command type ${record.type} is not defined`);
             }
+            if (record.duration !== 0) {
+                await segmentSink.write([record]);
+                return;
+            }
+            await persistPending(segmentSink, closer, pool, record);
         }
     };
 }

package/src/domain/segment/segmentCoalesce.js

@@ -0,0 +1,164 @@
+/**
+ * Returns the per-machine lane state, creating it when missing.
+ *
+ * @param {Map} lanes - machine → lane map
+ * @param {string} machine - machine id
+ * @returns {object} lane state
+ */
+function laneState(lanes, machine) {
+    if (!lanes.has(machine)) {
+        lanes.set(machine, { queue: Promise.resolve(), pending: null, timer: null });
+    }
+    return lanes.get(machine);
+}
+
+/**
+ * Builds the coalesce key for a pending heartbeat.
+ *
+ * @param {object} parsed - raw segment JSON
+ * @returns {string} coalesce key
+ */
+function heartbeatKey(parsed) {
+    const kind = typeof parsed.kind === 'string' && parsed.kind.length > 0 ? parsed.kind : 'phase';
+    return `${parsed.machine}\0${kind}\0${parsed.start}`;
+}
+
+/**
+ * Whether the payload is a pending open-segment heartbeat.
+ *
+ * @param {object} parsed - raw segment JSON
+ * @returns {boolean}
+ */
+function isHeartbeat(parsed) {
+    return parsed.type === 'segment' && parsed.duration === 0;
+}
+
+/**
+ * Flushes a buffered pending heartbeat batch through the codec.
+ *
+ * @param {object} lane - lane state
+ * @param {object} codec - downstream codec
+ * @returns {Promise<void>}
+ */
+async function flushLane(lane, codec) {
+    const batch = lane.pending;
+    if (!batch) {
+        return;
+    }
+    lane.pending = null;
+    if (lane.timer) {
+        clearTimeout(lane.timer);
+        lane.timer = null;
+    }
+    try {
+        await codec.accept({ destination: batch.destination, payload: batch.payload });
+        for (const settle of batch.settles) {
+            settle.ack();
+        }
+    } catch (error) {
+        for (const settle of batch.settles) {
+            settle.nack();
+        }
+        throw error;
+    }
+}
+
+/**
+ * Buffers or flushes one pending heartbeat on a lane.
+ *
+ * @param {object} ctx - lane context with api/size/intervalMs
+ * @param {object} envelope - STOMP envelope
+ * @param {object} parsed - parsed payload
+ * @returns {Promise<void>}
+ */
+async function bufferHeartbeat(ctx, envelope, parsed) {
+    const key = heartbeatKey(parsed);
+    const { api, lane, size, intervalMs } = ctx;
+    if (lane.pending && lane.pending.key === key) {
+        lane.pending.payload = envelope.payload;
+        lane.pending.settles.push(envelope.settle);
+    } else {
+        await api.flush(lane);
+        const pending = {
+            key,
+            destination: envelope.destination,
+            payload: envelope.payload,
+            settles: [envelope.settle]
+        };
+        const timer = setTimeout(() => {
+            lane.queue = lane.queue.then(() => {
+                return api.flush(lane);
+            });
+        }, intervalMs);
+        Object.assign(lane, { pending, timer });
+    }
+    if (lane.pending && lane.pending.settles.length >= size) {
+        await api.flush(lane);
+    }
+}
+
+/**
+ * Handles one envelope on a machine lane.
+ *
+ * @param {object} ctx - lane context
+ * @param {object} envelope - STOMP envelope with settle
+ * @param {object} parsed - parsed payload
+ * @returns {Promise<void>}
+ */
+async function step(ctx, envelope, parsed) {
+    if (isHeartbeat(parsed)) {
+        await bufferHeartbeat(ctx, envelope, parsed);
+        return;
+    }
+    await ctx.api.flush(ctx.lane);
+    try {
+        await ctx.codec.accept({ destination: envelope.destination, payload: envelope.payload });
+        envelope.settle.ack();
+    } catch (error) {
+        envelope.settle.nack();
+        throw error;
+    }
+}
+
+/**
+ * Coalesces pending segment heartbeats per machine before the codec.
+ *
+ * Expects STOMP envelopes with manual settle: `{ destination, payload, settle }`.
+ * Consecutive `type=segment` + `duration=0` messages that share
+ * `(machine, kind, start)` keep only the latest payload; all settles ack
+ * after one successful downstream accept. Retag, split, completed segments,
+ * and key changes flush immediately. Machines run on independent lanes.
+ *
+ * @param {object} codec - Downstream collector with accept({destination, payload})
+ * @param {object} [options] - Coalesce tuning
+ * @param {number} [options.size=50] - Max pending heartbeats before forced flush
+ * @param {number} [options.interval=2] - Flush interval in seconds
+ * @returns {object} Collector with accept(envelope)
+ *
+ * @example
+ *   const coalesce = segmentCoalesce(segmentCodec(dispatch), { size: 50, interval: 2 });
+ *   await coalesce.accept({ destination, payload, settle });
+ */
+export default function segmentCoalesce(codec, options = {}) {
+    if (!codec || typeof codec.accept !== 'function') {
+        throw new Error('Codec must have an accept() method');
+    }
+    const size = options.size || 50;
+    const intervalMs = (options.interval || 2) * 1000;
+    const lanes = new Map();
+    const api = {
+        accept(envelope) {
+            const parsed = JSON.parse(envelope.payload);
+            const lane = laneState(lanes, parsed.machine);
+            const ctx = { api, lane, codec, size, intervalMs };
+            lane.queue = lane.queue.then(() => {
+                return step(ctx, envelope, parsed);
+            });
+            return lane.queue;
+        },
+        async flush(lane) {
+            await flushLane(lane, codec);
+        }
+    };
+    return api;
+}

package/src/infrastructure/ingest/modbus/silentStreams.js

@@ -18,37 +18,37 @@ function isSilent(lastSeen, name, clk, budgetMs) {
 }
 
 /**
- * Stops one active poll if present.
+ * Stops one started poll if present. The poll instance is kept for reuse.
  *
- * @param {Map<string, object>} active - Active polls by name
+ * @param {object} state - Gate mutable state
  * @param {string} name - Stream id
  */
-function stopOne(active, name) {
-    const poll = active.get(name);
-    if (!poll) {
+function stopOne(state, name) {
+    if (!state.live.has(name)) {
         return;
     }
-    poll.stop();
-    active.delete(name);
+    state.held.get(name).stop();
+    state.live.delete(name);
 }
 
 /**
- * Starts one poll when not already active.
+ * Starts one poll, opening the source only the first time.
  *
- * @param {Map<string, object>} active - Active polls by name
+ * @param {object} state - Gate mutable state
  * @param {object} source - Stream source
- * @param {object} collector - Metrics collector
- * @param {object} clk - Clock
- * @param {number} intervalSec - Poll interval seconds
  */
-function startOne(active, source, collector, clk, intervalSec) {
+function startOne(state, source) {
     const name = source.name();
-    if (active.has(name)) {
+    if (state.live.has(name)) {
         return;
     }
-    const poll = source.open(collector, clk, intervalSec);
-    active.set(name, poll);
+    let poll = state.held.get(name);
+    if (!poll) {
+        poll = source.open(state.collector, state.clk, state.intervalSec);
+        state.held.set(name, poll);
+    }
     poll.start();
+    state.live.add(name);
 }
 
 /**
@@ -63,9 +63,9 @@ function pulse(state) {
     for (const source of state.sources) {
         const name = source.name();
         if (isSilent(state.lastSeen, name, state.clk, state.budgetMs)) {
-            startOne(state.active, source, state.collector, state.clk, state.intervalSec);
+            startOne(state, source);
         } else {
-            stopOne(state.active, name);
+            stopOne(state, name);
         }
     }
 }
@@ -88,7 +88,8 @@ function gateState(config) {
         }),
         clear: config.clear || clearInterval,
         lastSeen: new Map(),
-        active: new Map(),
+        held: new Map(),
+        live: new Set(),
         collector: undefined,
         timer: undefined,
         running: false
@@ -143,9 +144,10 @@ export default function silentStreams(config) {
                 state.clear(state.timer);
                 state.timer = undefined;
             }
-            for (const name of [...state.active.keys()]) {
-                stopOne(state.active, name);
+            for (const name of [...state.live]) {
+                stopOne(state, name);
             }
+            state.held.clear();
         },
         pulse: runPulse
     };

package/src/infrastructure/ingest/pipelines/segmentPipeline.js

@@ -3,6 +3,7 @@ import {
     stompSource
 } from '@yarkivaev/source-to-sink';
 import segmentDispatch from '../../../domain/segment/dispatch.js';
+import segmentCoalesce from '../../../domain/segment/segmentCoalesce.js';
 import silenceBudget from '../../../domain/segment/silenceBudget.js';
 import segmentCodec from '../codecs/segmentCodec.js';
 import silentOpenWatch from '../silentOpenWatch.js';
@@ -19,9 +20,27 @@ export const segmentsIngestDestination = '/queue/scada.segments.ingest';
 
 export { default as segmentDispatch } from '../../../domain/segment/dispatch.js';
 
+/**
+ * Builds segment and split postgres sinks that share one pool.
+ *
+ * @param {string} postgres - PostgreSQL URL
+ * @param {object} pool - shared pg pool
+ * @returns {{ segmentSink: object, splitSink: object }}
+ */
+function segmentSinks(postgres, pool) {
+    const shared = { pool, conflict: segmentConflict };
+    return {
+        segmentSink: postgresSink(postgres, 'segments', segmentColumns,
+            { ...shared, update: segmentUpdateColumns }),
+        splitSink: postgresSink(postgres, 'segments', segmentColumns,
+            { ...shared, update: splitUpdateColumns })
+    };
+}
+
 /**
  * Pipeline for streaming STOMP segment data to PostgreSQL segments table.
  * Subscribes to the durable ingest queue so shovel traffic survives consumer gaps.
+ * Pending heartbeats are coalesced per machine; machines run in parallel lanes.
  *
  * @param {string} stomp - STOMP broker URL
  * @param {string} postgres - PostgreSQL connection URL
@@ -30,21 +49,24 @@ export { default as segmentDispatch } from '../../../domain/segment/dispatch.js'
  * @returns {object} Pipeline with start() and stop() methods
  */
 export default function segmentPipeline(stomp, postgres, pool, config) {
-    const segmentSink = postgresSink(postgres, 'segments', segmentColumns,
-        { conflict: segmentConflict, update: segmentUpdateColumns });
-    const splitSink = postgresSink(postgres, 'segments', segmentColumns,
-        { conflict: segmentConflict, update: splitUpdateColumns });
-    const retag = retagSink(pool);
-    const closer = closeOrphanOpen(pool);
-    const dispatch = segmentDispatch(segmentSink, retag, splitSink, closer);
-    const codec = segmentCodec(dispatch);
+    const { segmentSink, splitSink } = segmentSinks(postgres, pool);
+    const dispatch = segmentDispatch(segmentSink, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
+    const coalesce = segmentCoalesce(segmentCodec(dispatch), {
+        size: config.size,
+        interval: config.interval
+    });
     const destination = config.segmentsDestination || segmentsIngestDestination;
-    const source = stompSource(stomp, destination, codec,
-        { login: config.login, passcode: config.passcode, host: config.host });
-    const window = config.segmentWindow || 15;
-    const budget = silenceBudget(window);
-    const pollMs = (config.poll || 5) * 1000;
-    const silence = silentOpenWatch(closeSilentOpen(pool), budget, { intervalMs: pollMs });
+    const source = stompSource(stomp, destination, coalesce, {
+        login: config.login,
+        passcode: config.passcode,
+        host: config.host,
+        serial: false,
+        manualAck: true
+    });
+    const budget = silenceBudget(config.segmentWindow || 15);
+    const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
+        intervalMs: (config.poll || 5) * 1000
+    });
     return {
         destination,
         start() {

package/src/infrastructure/ingest/sinks/closeOrphanOpen.js

@@ -4,7 +4,7 @@ import processingErrorLog from '../processingErrorLog.js';
  * Closes stale open segment rows for one machine before a new open segment lands.
  *
  * @param {object} pool - pg Pool with query() method
- * @returns {object} closer with close(machine, startTime, kind)
+ * @returns {object} closer with close(machine, startTime, kind, client?)
  *
  * @example
  *   const closer = closeOrphanOpen(pool);
@@ -15,10 +15,11 @@ export default function closeOrphanOpen(pool) {
         throw new Error('Pool must have a query() method');
     }
     return {
-        async close(machine, startTime, kind) {
+        async close(machine, startTime, kind, client) {
             const track = kind || 'phase';
+            const runner = client && typeof client.query === 'function' ? client : pool;
             try {
-                await pool.query(
+                await runner.query(
                     `UPDATE segments
                      SET end_time = start_time + interval '1 second', duration = 1
                      WHERE machine = $1 AND kind = $3 AND duration = 0 AND start_time <> $2`,