@yarkivaev/scada 2.3.77 → 2.3.79

package/db/migrations/V0008__segments_updated_at.sql

@@ -0,0 +1 @@
+ALTER TABLE segments ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.77",
+  "version": "2.3.79",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",

package/src/application/supervisorSink.js

@@ -87,7 +87,7 @@ export default function supervisorSink(env = process.env) {
             } finally {
                 await adminPool.end();
             }
-            segments.start();
+            await segments.start();
             decisions.start();
             alerts.start();
             if (httpServer) {

package/src/domain/segment/normalize.js

@@ -27,6 +27,7 @@ export default function segmentNormalize(parsed) {
         options: parsed.options ? JSON.stringify(parsed.options) : null,
         tags: parsed.tags ? JSON.stringify(parsed.tags) : null,
         properties: parsed.properties ? JSON.stringify(parsed.properties) : null,
-        resolved
+        resolved,
+        updated_at: new Date().toISOString()
     };
 }

package/src/domain/segment/segmentContinue.js

@@ -0,0 +1,31 @@
+/**
+ * Pulls a later fragment of the same segment back onto the previous row
+ * when the uncovered gap is still inside the segmentation continuity budget.
+ * A supervisor restart then does not open a hole where metric samples continued.
+ *
+ * @param {number} budgetSeconds - maximum uncovered gap that stays one segment
+ * @returns {function} apply(record, prior) -> record
+ *
+ * @example
+ *   const apply = segmentContinue(30);
+ *   const row = apply(incoming, prior);
+ */
+export default function segmentContinue(budgetSeconds) {
+    return function apply(record, prior) {
+        if (!prior || prior.name !== record.name) {
+            return record;
+        }
+        const priorStart = new Date(prior.start_time).toISOString();
+        if (priorStart === record.start_time) {
+            return record;
+        }
+        const gap = Date.parse(record.start_time) - Date.parse(prior.end_time);
+        if (gap < 0 || gap > budgetSeconds * 1000) {
+            return record;
+        }
+        const startMs = Date.parse(priorStart);
+        const endMs = Date.parse(record.end_time);
+        const duration = record.duration === 0 ? 0 : Math.max(1, (endMs - startMs) / 1000);
+        return { ...record, start_time: priorStart, duration };
+    };
+}

package/src/infrastructure/ingest/pipelines/segmentPipeline.js

@@ -6,17 +6,20 @@ import segmentDispatch from '../../../domain/segment/dispatch.js';
 import segmentCoalesce from '../../../domain/segment/segmentCoalesce.js';
 import silenceBudget from '../../../domain/segment/silenceBudget.js';
 import segmentCodec from '../codecs/segmentCodec.js';
+import processingErrorLog from '../processingErrorLog.js';
 import silentOpenWatch from '../silentOpenWatch.js';
+import abutSegments from '../sinks/abutSegments.js';
 import closeOrphanOpen from '../sinks/closeOrphanOpen.js';
 import closeSilentOpen from '../sinks/closeSilentOpen.js';
+import continuedSegments from '../sinks/continuedSegments.js';
 import retagSink from '../sinks/retagSink.js';
 
 export const segmentColumns = ['machine', 'kind', 'name', 'start_time', 'end_time', 'duration',
-                               'options', 'tags', 'properties', 'resolved'];
+                               'options', 'tags', 'properties', 'resolved', 'updated_at'];
 export const segmentConflict = ['machine', 'kind', 'start_time'];
-export const segmentUpdateColumns = ['name', 'end_time', 'duration', 'options', 'resolved'];
+export const segmentUpdateColumns = ['name', 'end_time', 'duration', 'options', 'resolved', 'updated_at'];
 export const splitUpdateColumns = ['name', 'end_time', 'duration', 'tags', 'options', 'resolved'];
-export const segmentUpdateWhere = 'segments.duration = 0 OR EXCLUDED.duration > 0';
+export const segmentUpdateWhere = 'segments.duration = 0 OR EXCLUDED.duration > 0 OR EXCLUDED.end_time > segments.end_time';
 export const segmentsIngestDestination = '/queue/scada.segments.ingest';
 
 export { default as segmentDispatch } from '../../../domain/segment/dispatch.js';
@@ -51,8 +54,10 @@ function segmentSinks(postgres, pool) {
  * @returns {object} Pipeline with start() and stop() methods
  */
 export default function segmentPipeline(stomp, postgres, pool, config) {
+    const budget = silenceBudget(config.segmentWindow || 15);
     const { segmentSink, splitSink } = segmentSinks(postgres, pool);
-    const dispatch = segmentDispatch(segmentSink, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
+    const continued = continuedSegments(segmentSink, pool, budget);
+    const dispatch = segmentDispatch(continued, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
     const coalesce = segmentCoalesce(segmentCodec(dispatch), {
         size: config.size,
         interval: config.interval
@@ -65,15 +70,18 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
         serial: false,
         manualAck: true
     });
-    const budget = silenceBudget(config.segmentWindow || 15);
     const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
         intervalMs: (config.poll || 5) * 1000
     });
     return {
         destination,
-        start() {
+        async start() {
+            const repair = abutSegments(pool, budget);
+            void repair.catch((error) => {
+                processingErrorLog('abut_segments', error, { budgetSeconds: budget });
+            });
             source.start();
-            void silence.start();
+            await silence.start();
         },
         stop() {
             silence.stop();

package/src/infrastructure/ingest/silentOpenWatch.js

@@ -18,14 +18,27 @@ export default function silentOpenWatch(closer, budget, options) {
         throw new Error(`Budget must be a positive number: ${budget}`);
     }
     const intervalMs = options && options.intervalMs ? options.intervalMs : 5000;
+    const gate = { busy: false };
     let timer;
     let active = false;
+    function claim() {
+        const free = !gate.busy;
+        gate.busy = true;
+        return free;
+    }
+    function release() {
+        gate.busy = false;
+    }
     async function tick() {
+        if (!claim()) {
+            return;
+        }
         try {
             await closer.close(budget);
         } catch {
             /* next tick retries */
         }
+        release();
     }
     return {
         async start() {

package/src/infrastructure/ingest/sinks/abutSegments.js

@@ -0,0 +1,33 @@
+/**
+ * Closes uncovered time between adjacent same-name segments that stayed
+ * inside the continuity budget. Existing restart fragments then meet on
+ * the timeline without deleting rows.
+ *
+ * @param {object} pool - pg pool
+ * @param {number} budgetSeconds - maximum gap that still counts as continuous
+ * @returns {Promise<void>}
+ *
+ * @example
+ *   await abutSegments(pool, 30);
+ */
+export default async function abutSegments(pool, budgetSeconds) {
+    await pool.query(
+        `UPDATE segments AS left_row
+         SET end_time = right_row.start_time,
+             duration = EXTRACT(EPOCH FROM (right_row.start_time - left_row.start_time))
+         FROM LATERAL (
+             SELECT next.start_time, next.name
+             FROM segments AS next
+             WHERE next.machine = left_row.machine
+               AND next.kind = left_row.kind
+               AND next.start_time > left_row.start_time
+             ORDER BY next.start_time
+             LIMIT 1
+         ) AS right_row
+         WHERE left_row.duration > 0
+           AND left_row.name = right_row.name
+           AND right_row.start_time > left_row.end_time
+           AND right_row.start_time - left_row.end_time <= make_interval(secs => $1)`,
+        [budgetSeconds]
+    );
+}

package/src/infrastructure/ingest/sinks/closeSilentOpen.js

@@ -1,7 +1,8 @@
 import processingErrorLog from '../processingErrorLog.js';
 
 /**
- * Closes open segment rows whose advanced end boundary is older than a wall-clock silence budget.
+ * Closes open segment rows that have not been written for a wall-clock silence budget.
+ * Freshness is updated_at, so replaying historical samples does not look idle.
  * Requires end_time > start_time so a lone Started (end equals start) is never treated as silence.
  * Does not insert a replacement Started row.
  *
@@ -27,7 +28,7 @@ export default function closeSilentOpen(pool) {
                          )
                      WHERE duration = 0
                        AND end_time > start_time
-                       AND end_time < NOW() - make_interval(secs => $1)`,
+                       AND updated_at < NOW() - make_interval(secs => $1)`,
                     [budgetSeconds]
                 );
             } catch (error) {

package/src/infrastructure/ingest/sinks/continuedSegments.js

@@ -0,0 +1,49 @@
+import segmentContinue from '../../../domain/segment/segmentContinue.js';
+
+/**
+ * Loads the latest segment row visible to this write.
+ *
+ * @param {object} runner - pg client or pool
+ * @param {object} record - incoming segment record
+ * @returns {Promise<object|undefined>} latest row
+ */
+async function latest(runner, record) {
+    const result = await runner.query(
+        `SELECT name, start_time, end_time
+         FROM segments
+         WHERE machine = $1 AND kind = $2
+         ORDER BY start_time DESC
+         LIMIT 1`,
+        [record.machine, record.kind || 'phase']
+    );
+    return result.rows[0];
+}
+
+/**
+ * Segment sink that continues the previous row across a restart
+ * when metric samples never left the continuity budget.
+ *
+ * @param {object} inner - postgres segment sink
+ * @param {object} pool - pg pool used when no transaction client is passed
+ * @param {number} budgetSeconds - segmentation continuity budget
+ * @returns {object} sink with write(records, client)
+ *
+ * @example
+ *   const sink = continuedSegments(segmentSink, pool, 30);
+ *   await sink.write([record], client);
+ */
+export default function continuedSegments(inner, pool, budgetSeconds) {
+    const apply = segmentContinue(budgetSeconds);
+    return {
+        async write(records, client) {
+            const runner = client && typeof client.query === 'function' ? client : pool;
+            const next = await records.reduce(async (prior, record) => {
+                const rows = await prior;
+                const previous = await latest(runner, record);
+                rows.push(apply(record, previous));
+                return rows;
+            }, Promise.resolve([]));
+            return inner.write(next, client);
+        }
+    };
+}