@yarkivaev/scada 2.3.57 → 2.3.59

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.57",
+  "version": "2.3.59",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",
@@ -29,7 +29,7 @@
   "dependencies": {
     "@clickhouse/client": "^1.0.0",
     "@yarkivaev/simple-server": "1.0.1",
-    "@yarkivaev/source-to-sink": "1.3.3",
+    "@yarkivaev/source-to-sink": "1.3.4",
     "amqplib": "^0.10.9",
     "pg": "^8.0.0",
     "stompit": "^1.0.0"

package/src/domain/segment/dispatch.js

@@ -1,31 +1,69 @@
+/**
+ * Persists a pending heartbeat, optionally inside one pool transaction.
+ *
+ * @param {object} segmentSink - upsert sink
+ * @param {object} closer - orphan closer
+ * @param {object} [pool] - optional pg pool
+ * @param {object} record - normalized segment record
+ * @returns {Promise<void>}
+ */
+async function persistPending(segmentSink, closer, pool, record) {
+    const kind = record.kind || 'phase';
+    if (!pool || typeof pool.connect !== 'function') {
+        await closer.close(record.machine, record.start_time, kind);
+        await segmentSink.write([record]);
+        return;
+    }
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        await closer.close(record.machine, record.start_time, kind, client);
+        await segmentSink.write([record], client);
+        await client.query('COMMIT');
+    } catch (error) {
+        await client.query('ROLLBACK').catch((ignored) => { return ignored; });
+        throw error;
+    } finally {
+        client.release();
+    }
+}
+
 /**
  * Routes normalized segment records to the correct persistence sink.
  *
+ * When a pool is supplied, pending segment heartbeats (`duration === 0`)
+ * run orphan-close and upsert inside one transaction on a single client.
+ *
  * @param {object} segmentSink - batch insert/update sink
  * @param {object} retag - retag sink
  * @param {object} splitSink - split update sink
  * @param {object} closer - orphan open segment closer
+ * @param {object} [pool] - optional pg pool for transactional pending writes
  * @returns {object} collector with accept(record)
  *
  * @example
- *   const route = segmentDispatch(segmentSink, retag, splitSink, closer);
+ *   const route = segmentDispatch(segmentSink, retag, splitSink, closer, pool);
  *   await route.accept({ type: 'segment', machine: 'm1', start_time: '...', duration: 0 });
  */
-export default function segmentDispatch(segmentSink, retag, splitSink, closer) {
+export default function segmentDispatch(segmentSink, retag, splitSink, closer, pool) {
     return {
         async accept(record) {
             if (record.type === 'retag') {
                 await retag.accept(record);
-            } else if (record.type === 'split') {
+                return;
+            }
+            if (record.type === 'split') {
                 await splitSink.write([record]);
-            } else if (record.type === 'segment') {
-                if (record.duration === 0) {
-                    await closer.close(record.machine, record.start_time, record.kind || 'phase');
-                }
-                await segmentSink.write([record]);
-            } else {
+                return;
+            }
+            if (record.type !== 'segment') {
                 throw new Error(`Segment command type ${record.type} is not defined`);
             }
+            if (record.duration !== 0) {
+                await segmentSink.write([record]);
+                return;
+            }
+            await persistPending(segmentSink, closer, pool, record);
         }
     };
 }

package/src/domain/segment/segmentCoalesce.js

@@ -0,0 +1,164 @@
+/**
+ * Returns the per-machine lane state, creating it when missing.
+ *
+ * @param {Map} lanes - machine → lane map
+ * @param {string} machine - machine id
+ * @returns {object} lane state
+ */
+function laneState(lanes, machine) {
+    if (!lanes.has(machine)) {
+        lanes.set(machine, { queue: Promise.resolve(), pending: null, timer: null });
+    }
+    return lanes.get(machine);
+}
+
+/**
+ * Builds the coalesce key for a pending heartbeat.
+ *
+ * @param {object} parsed - raw segment JSON
+ * @returns {string} coalesce key
+ */
+function heartbeatKey(parsed) {
+    const kind = typeof parsed.kind === 'string' && parsed.kind.length > 0 ? parsed.kind : 'phase';
+    return `${parsed.machine}\0${kind}\0${parsed.start}`;
+}
+
+/**
+ * Whether the payload is a pending open-segment heartbeat.
+ *
+ * @param {object} parsed - raw segment JSON
+ * @returns {boolean}
+ */
+function isHeartbeat(parsed) {
+    return parsed.type === 'segment' && parsed.duration === 0;
+}
+
+/**
+ * Flushes a buffered pending heartbeat batch through the codec.
+ *
+ * @param {object} lane - lane state
+ * @param {object} codec - downstream codec
+ * @returns {Promise<void>}
+ */
+async function flushLane(lane, codec) {
+    const batch = lane.pending;
+    if (!batch) {
+        return;
+    }
+    lane.pending = null;
+    if (lane.timer) {
+        clearTimeout(lane.timer);
+        lane.timer = null;
+    }
+    try {
+        await codec.accept({ destination: batch.destination, payload: batch.payload });
+        for (const settle of batch.settles) {
+            settle.ack();
+        }
+    } catch (error) {
+        for (const settle of batch.settles) {
+            settle.nack();
+        }
+        throw error;
+    }
+}
+
+/**
+ * Buffers or flushes one pending heartbeat on a lane.
+ *
+ * @param {object} ctx - lane context with api/size/intervalMs
+ * @param {object} envelope - STOMP envelope
+ * @param {object} parsed - parsed payload
+ * @returns {Promise<void>}
+ */
+async function bufferHeartbeat(ctx, envelope, parsed) {
+    const key = heartbeatKey(parsed);
+    const { api, lane, size, intervalMs } = ctx;
+    if (lane.pending && lane.pending.key === key) {
+        lane.pending.payload = envelope.payload;
+        lane.pending.settles.push(envelope.settle);
+    } else {
+        await api.flush(lane);
+        const pending = {
+            key,
+            destination: envelope.destination,
+            payload: envelope.payload,
+            settles: [envelope.settle]
+        };
+        const timer = setTimeout(() => {
+            lane.queue = lane.queue.then(() => {
+                return api.flush(lane);
+            });
+        }, intervalMs);
+        Object.assign(lane, { pending, timer });
+    }
+    if (lane.pending && lane.pending.settles.length >= size) {
+        await api.flush(lane);
+    }
+}
+
+/**
+ * Handles one envelope on a machine lane.
+ *
+ * @param {object} ctx - lane context
+ * @param {object} envelope - STOMP envelope with settle
+ * @param {object} parsed - parsed payload
+ * @returns {Promise<void>}
+ */
+async function step(ctx, envelope, parsed) {
+    if (isHeartbeat(parsed)) {
+        await bufferHeartbeat(ctx, envelope, parsed);
+        return;
+    }
+    await ctx.api.flush(ctx.lane);
+    try {
+        await ctx.codec.accept({ destination: envelope.destination, payload: envelope.payload });
+        envelope.settle.ack();
+    } catch (error) {
+        envelope.settle.nack();
+        throw error;
+    }
+}
+
+/**
+ * Coalesces pending segment heartbeats per machine before the codec.
+ *
+ * Expects STOMP envelopes with manual settle: `{ destination, payload, settle }`.
+ * Consecutive `type=segment` + `duration=0` messages that share
+ * `(machine, kind, start)` keep only the latest payload; all settles ack
+ * after one successful downstream accept. Retag, split, completed segments,
+ * and key changes flush immediately. Machines run on independent lanes.
+ *
+ * @param {object} codec - Downstream collector with accept({destination, payload})
+ * @param {object} [options] - Coalesce tuning
+ * @param {number} [options.size=50] - Max pending heartbeats before forced flush
+ * @param {number} [options.interval=2] - Flush interval in seconds
+ * @returns {object} Collector with accept(envelope)
+ *
+ * @example
+ *   const coalesce = segmentCoalesce(segmentCodec(dispatch), { size: 50, interval: 2 });
+ *   await coalesce.accept({ destination, payload, settle });
+ */
+export default function segmentCoalesce(codec, options = {}) {
+    if (!codec || typeof codec.accept !== 'function') {
+        throw new Error('Codec must have an accept() method');
+    }
+    const size = options.size || 50;
+    const intervalMs = (options.interval || 2) * 1000;
+    const lanes = new Map();
+    const api = {
+        accept(envelope) {
+            const parsed = JSON.parse(envelope.payload);
+            const lane = laneState(lanes, parsed.machine);
+            const ctx = { api, lane, codec, size, intervalMs };
+            lane.queue = lane.queue.then(() => {
+                return step(ctx, envelope, parsed);
+            });
+            return lane.queue;
+        },
+        async flush(lane) {
+            await flushLane(lane, codec);
+        }
+    };
+    return api;
+}

package/src/infrastructure/ingest/pipelines/segmentPipeline.js

@@ -3,6 +3,7 @@ import {
     stompSource
 } from '@yarkivaev/source-to-sink';
 import segmentDispatch from '../../../domain/segment/dispatch.js';
+import segmentCoalesce from '../../../domain/segment/segmentCoalesce.js';
 import silenceBudget from '../../../domain/segment/silenceBudget.js';
 import segmentCodec from '../codecs/segmentCodec.js';
 import silentOpenWatch from '../silentOpenWatch.js';
@@ -19,9 +20,27 @@ export const segmentsIngestDestination = '/queue/scada.segments.ingest';
 
 export { default as segmentDispatch } from '../../../domain/segment/dispatch.js';
 
+/**
+ * Builds segment and split postgres sinks that share one pool.
+ *
+ * @param {string} postgres - PostgreSQL URL
+ * @param {object} pool - shared pg pool
+ * @returns {{ segmentSink: object, splitSink: object }}
+ */
+function segmentSinks(postgres, pool) {
+    const shared = { pool, conflict: segmentConflict };
+    return {
+        segmentSink: postgresSink(postgres, 'segments', segmentColumns,
+            { ...shared, update: segmentUpdateColumns }),
+        splitSink: postgresSink(postgres, 'segments', segmentColumns,
+            { ...shared, update: splitUpdateColumns })
+    };
+}
+
 /**
  * Pipeline for streaming STOMP segment data to PostgreSQL segments table.
  * Subscribes to the durable ingest queue so shovel traffic survives consumer gaps.
+ * Pending heartbeats are coalesced per machine; machines run in parallel lanes.
  *
  * @param {string} stomp - STOMP broker URL
  * @param {string} postgres - PostgreSQL connection URL
@@ -30,21 +49,24 @@ export { default as segmentDispatch } from '../../../domain/segment/dispatch.js'
  * @returns {object} Pipeline with start() and stop() methods
  */
 export default function segmentPipeline(stomp, postgres, pool, config) {
-    const segmentSink = postgresSink(postgres, 'segments', segmentColumns,
-        { conflict: segmentConflict, update: segmentUpdateColumns });
-    const splitSink = postgresSink(postgres, 'segments', segmentColumns,
-        { conflict: segmentConflict, update: splitUpdateColumns });
-    const retag = retagSink(pool);
-    const closer = closeOrphanOpen(pool);
-    const dispatch = segmentDispatch(segmentSink, retag, splitSink, closer);
-    const codec = segmentCodec(dispatch);
+    const { segmentSink, splitSink } = segmentSinks(postgres, pool);
+    const dispatch = segmentDispatch(segmentSink, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
+    const coalesce = segmentCoalesce(segmentCodec(dispatch), {
+        size: config.size,
+        interval: config.interval
+    });
     const destination = config.segmentsDestination || segmentsIngestDestination;
-    const source = stompSource(stomp, destination, codec,
-        { login: config.login, passcode: config.passcode, host: config.host });
-    const window = config.segmentWindow || 15;
-    const budget = silenceBudget(window);
-    const pollMs = (config.poll || 5) * 1000;
-    const silence = silentOpenWatch(closeSilentOpen(pool), budget, { intervalMs: pollMs });
+    const source = stompSource(stomp, destination, coalesce, {
+        login: config.login,
+        passcode: config.passcode,
+        host: config.host,
+        serial: false,
+        manualAck: true
+    });
+    const budget = silenceBudget(config.segmentWindow || 15);
+    const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
+        intervalMs: (config.poll || 5) * 1000
+    });
     return {
         destination,
         start() {

package/src/infrastructure/ingest/sinks/closeOrphanOpen.js

@@ -4,7 +4,7 @@ import processingErrorLog from '../processingErrorLog.js';
  * Closes stale open segment rows for one machine before a new open segment lands.
  *
  * @param {object} pool - pg Pool with query() method
- * @returns {object} closer with close(machine, startTime, kind)
+ * @returns {object} closer with close(machine, startTime, kind, client?)
  *
  * @example
  *   const closer = closeOrphanOpen(pool);
@@ -15,10 +15,11 @@ export default function closeOrphanOpen(pool) {
         throw new Error('Pool must have a query() method');
     }
     return {
-        async close(machine, startTime, kind) {
+        async close(machine, startTime, kind, client) {
             const track = kind || 'phase';
+            const runner = client && typeof client.query === 'function' ? client : pool;
             try {
-                await pool.query(
+                await runner.query(
                     `UPDATE segments
                      SET end_time = start_time + interval '1 second', duration = 1
                      WHERE machine = $1 AND kind = $3 AND duration = 0 AND start_time <> $2`,

package/src/infrastructure/persistence/memory/operations.js

@@ -99,7 +99,7 @@ function filterLatest(store, machineId, kind, bound) {
  * In-memory operations state port for tests and local runs.
  *
  * @param {object} store - shared mutable store with operations array
- * @returns {object} operations port matching operationStatePg shape
+ * @returns {object} operations port matching operationStatePg shape including drop
  */
 export default function operationStateMemory(store) {
     if (!store.operations) {
@@ -126,6 +126,15 @@ export default function operationStateMemory(store) {
             const [row] = store.operations.splice(index, 1);
             return Promise.resolve(row);
         },
+        drop(machineId, key) {
+            const index = store.operations.findIndex((row) => {
+                return row.machine === machineId && row.key === key;
+            });
+            if (index >= 0) {
+                store.operations.splice(index, 1);
+            }
+            return Promise.resolve();
+        },
         listForMachine(machineId, kind, range) {
             return Promise.resolve(filterList(store, machineId, kind, range));
         },

package/src/infrastructure/persistence/pg/operations.js

@@ -120,17 +120,33 @@ async function removeOperation(pool, machineId, key) {
     return result.rows[0];
 }
 
+/**
+ * Deletes an operation when present. Absence is success.
+ *
+ * @param {object} pool - pg pool
+ * @param {string} machineId - machine identifier
+ * @param {string} key - operation storage key
+ * @returns {Promise<void>}
+ */
+function dropOperation(pool, machineId, key) {
+    return pool.query(
+        'DELETE FROM operations WHERE machine = $1 AND key = $2',
+        [machineId, key]
+    );
+}
+
 /**
  * PostgreSQL operations persistence port for generic machine operations.
  *
  * @param {object} pool - pg pool
- * @returns {object} operations port with upsert, get, remove, listForMachine, latestForMachine
+ * @returns {object} operations port with upsert, get, remove, drop, listForMachine, latestForMachine
  *
  * @example
  *   const store = operationStatePg(pool);
  *   await store.upsert({ machine: 'm1', key: 'nb-1', kind: 'sample', ... });
  *   await store.latestForMachine('m1', 'sample', { to: new Date() });
  *   await store.remove('m1', 'nb-1');
+ *   await store.drop('m1', 'nb-1');
  */
 export default function operationStatePg(pool) {
     return {
@@ -143,6 +159,9 @@ export default function operationStatePg(pool) {
         remove(machineId, key) {
             return removeOperation(pool, machineId, key);
         },
+        drop(machineId, key) {
+            return dropOperation(pool, machineId, key);
+        },
         listForMachine(machineId, kind, range) {
             return listForMachine(pool, machineId, kind, range);
         },

package/src/infrastructure/sync/operationSyncSink.js

@@ -3,7 +3,9 @@ import processingErrorLog from '../ingest/processingErrorLog.js';
 /**
  * PostgreSQL sink for generic operation sync records.
  *
- * @param {object} operations - Operations port with upsert(item) and remove(machineId, key)
+ * Federated deletes call operations.drop, which is idempotent when the row is absent.
+ *
+ * @param {object} operations - Operations port with upsert(item) and drop(machineId, key)
  * @returns {object} Sink with accept() and remove() methods
  *
  * @example
@@ -26,7 +28,7 @@ export default function operationSyncSink(operations) {
         },
         async remove(record) {
             try {
-                await operations.remove(record.machine, record.key);
+                await operations.drop(record.machine, record.key);
             } catch (error) {
                 processingErrorLog('operation_sync_sink', error, {
                     machine: record.machine,