@yarkivaev/scada 2.3.72 → 2.3.74

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.72",
+  "version": "2.3.74",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",

package/src/infrastructure/http/plant/routes/measurementRead.js

@@ -4,7 +4,7 @@
  * @param {object} sensor - machine sensor
  * @param {{start: Date, end: Date}} range - query window
  * @param {number} step - downsample step in ms
- * @param {string} [mode] - rises or changes
+ * @param {string} [mode] - rises, changes, or holds
  * @returns {Promise<Array>} series rows
  */
 export default function measurementRead(sensor, range, step, mode) {
@@ -14,6 +14,9 @@ export default function measurementRead(sensor, range, step, mode) {
     if (mode === 'changes') {
         return sensor.changes(range);
     }
+    if (mode === 'holds') {
+        return sensor.holds(range, step);
+    }
     return sensor.measurements(range, step);
 }
 
@@ -28,10 +31,14 @@ export default function measurementRead(sensor, range, step, mode) {
 export function measurementItem(key, sensor, rows) {
     const unit = rows.length > 0 ? rows[0].unit : '';
     const values = rows.map((row) => {
-        return {
+        const point = {
             timestamp: row.timestamp.toISOString(),
             value: row.value
         };
+        if (row.until) {
+            point.until = row.until.toISOString();
+        }
+        return point;
     });
     return { key, name: sensor.name(), unit, values };
 }

package/src/infrastructure/persistence/clickhouse/holds.js

@@ -0,0 +1,75 @@
+/**
+ * Coverage holds of one ClickHouse metric.
+ *
+ * Heartbeat repeats stay in one span. A value change or a silence
+ * longer than gap closes the span. Samples before the window are ignored.
+ *
+ * @example
+ *   await clickhouseHolds(conn, 'm1/diam', { start, end }, 3000, 'mm');
+ *
+ * @param {object} connection - ClickHouse connection with query method
+ * @param {string} topic - metric topic
+ * @param {{start: Date, end: Date}} range - half-open window
+ * @param {number} gap - maximum silence in milliseconds
+ * @param {string} unit - measurement unit
+ * @returns {Promise<Array<{timestamp: Date, value: number, until: Date, unit: string}>>} holds
+ */
+function formatDateTime(date) {
+    return date.toISOString().replace('Z', '').replace('T', ' ');
+}
+
+const HOLDS_SQL = `WITH windowed AS (
+    SELECT ts, value
+    FROM scada.metrics
+    WHERE topic = {topic:String}
+      AND ts >= {start:DateTime64(3)}
+      AND ts < {end:DateTime64(3)}
+), lagged AS (
+    SELECT
+        ts,
+        value,
+        lagInFrame(ts, 1) OVER (ORDER BY ts) AS prev_ts,
+        lagInFrame(toNullable(value), 1) OVER (ORDER BY ts) AS prev
+    FROM windowed
+), flagged AS (
+    SELECT
+        ts,
+        value,
+        if(
+            prev IS NULL
+                OR value != prev
+                OR toUnixTimestamp64Milli(ts) - toUnixTimestamp64Milli(prev_ts) > {gap:UInt64},
+            1,
+            0
+        ) AS fresh
+    FROM lagged
+), grouped AS (
+    SELECT
+        ts,
+        value,
+        sum(fresh) OVER (ORDER BY ts) AS grp
+    FROM flagged
+)
+SELECT min(ts) AS ts, max(ts) AS until, any(value) AS value
+FROM grouped
+GROUP BY grp
+ORDER BY ts`;
+
+export default async function clickhouseHolds(connection, topic, range, gap, unit) {
+    const rows = await connection.query(HOLDS_SQL, {
+        topic,
+        start: formatDateTime(range.start),
+        end: formatDateTime(range.end),
+        gap: Math.max(1, Number(gap) || 1000)
+    });
+    return rows.map((row) => {
+        return {
+            timestamp: new Date(`${row.ts}Z`),
+            value: row.value,
+            until: new Date(`${row.until}Z`),
+            unit
+        };
+    });
+}
+
+export { HOLDS_SQL };

package/src/infrastructure/persistence/clickhouse/rises.js

@@ -37,7 +37,7 @@ const RISES_SQL = `WITH prior AS (
         lagInFrame(toNullable(value), 1) OVER (ORDER BY ts) AS prev
     FROM series
 )
-SELECT ts, sum(1) OVER (ORDER BY ts) AS value
+SELECT ts, sum(1) OVER (ORDER BY ts) AS prefix
 FROM lagged
 WHERE ts >= {start:DateTime64(3)} AND prev = 0 AND value = 1
 ORDER BY ts`;
@@ -49,7 +49,7 @@ export default async function clickhouseRises(connection, topic, range, unit) {
         end: formatDateTime(range.end)
     });
     return rows.map((row) => {
-        return { timestamp: new Date(`${row.ts}Z`), value: row.value, unit };
+        return { timestamp: new Date(`${row.ts}Z`), value: Number(row.prefix), unit };
     });
 }
 

package/src/infrastructure/persistence/clickhouse/sensor.js

@@ -23,6 +23,7 @@
 import clickhouseStreamHub from './streamHub.js';
 import clickhouseRises from './rises.js';
 import clickhouseChanges from './changes.js';
+import clickhouseHolds from './holds.js';
 import clickhouseSeries from './series.js';
 import clickhouseLatest from './latest.js';
 
@@ -43,6 +44,9 @@ export default function clickhouseSensor(connection, topic, displayName, unit) {
         changes(range) {
             return clickhouseChanges(connection, topic, range, unit);
         },
+        holds(range, step) {
+            return clickhouseHolds(connection, topic, range, step, unit);
+        },
         stream(since, step, callback, clock) {
             return clickhouseStreamHub(connection).watch(topic, since, step, callback, {
                 unit,

package/src/infrastructure/persistence/coverageHolds.js

@@ -0,0 +1,39 @@
+/**
+ * Folds heartbeat samples into coverage holds.
+ *
+ * A new span starts on the first in-window sample, a value change,
+ * or a silence longer than gap. Samples before the window are ignored.
+ *
+ * @param {Array<{ts: Date, value: *}>} rows - raw metric rows
+ * @param {number} start - window start epoch milliseconds
+ * @param {number} end - exclusive window end epoch milliseconds
+ * @param {number} gap - maximum silence in milliseconds
+ * @returns {Array<{ts: Date, value: *, until: Date}>} holds
+ *
+ * @example
+ *   coverageHolds(rows, from, to, 3000);
+ */
+export default function coverageHolds(rows, start, end, gap) {
+    const series = (rows || []).filter((row) => {
+        const ts = new Date(row.ts).getTime();
+        return ts >= start && ts < end;
+    }).sort((left, right) => {
+        return new Date(left.ts) - new Date(right.ts);
+    });
+    const spans = [];
+    series.forEach((row) => {
+        const ts = new Date(row.ts);
+        const last = spans[spans.length - 1];
+        if (!last) {
+            spans.push({ ts, value: row.value, until: ts });
+            return;
+        }
+        const silence = ts.getTime() - last.until.getTime();
+        if (row.value !== last.value || silence > gap) {
+            spans.push({ ts, value: row.value, until: ts });
+            return;
+        }
+        last.until = ts;
+    });
+    return spans;
+}

package/src/infrastructure/persistence/memory/metrics.js

@@ -1,5 +1,6 @@
 import risingPrefixes from '../risingPrefixes.js';
 import levelChanges from '../levelChanges.js';
+import coverageHolds from '../coverageHolds.js';
 import { seriesWindow, topicRows } from '../topicWindow.js';
 
 function bucketKey(tsMs, originMs, stepMs) {
@@ -49,6 +50,15 @@ function changesOf(rows, topic, startIso, endIso) {
     });
 }
 
+function holdsOf(rows, topic, startIso, endIso, stepMs) {
+    const start = new Date(startIso).getTime();
+    const end = new Date(endIso).getTime();
+    const hits = rows.filter((row) => {
+        return row.topic === topic;
+    });
+    return coverageHolds(hits, start, end, Math.max(1, Number(stepMs) || 1000));
+}
+
 /**
  * In-memory metrics state port for tests and local runs.
  *
@@ -77,6 +87,9 @@ export default function metricsStateMemory(store) {
         changesForTopic(topic, startIso, endIso) {
             return changesOf(store.metrics, topic, startIso, endIso);
         },
+        holdsForTopic(topic, startIso, endIso, stepMs) {
+            return holdsOf(store.metrics, topic, startIso, endIso, stepMs);
+        },
         pollTopic(topic, afterIso, untilIso) {
             const after = new Date(afterIso).getTime();
             const until = new Date(untilIso).getTime();

package/src/infrastructure/persistence/memory/metricsDisabled.js

@@ -15,6 +15,9 @@ export default function metricsStateDisabled() {
         changesForTopic() {
             return Promise.resolve([]);
         },
+        holdsForTopic() {
+            return Promise.resolve([]);
+        },
         pollTopic() {
             return Promise.resolve([]);
         },

package/src/infrastructure/persistence/metricsSensor.js

@@ -49,6 +49,9 @@ export function pgMetricsRead(metrics) {
         changes(topic, startIso, endIso) {
             return metrics.changesForTopic(topic, startIso, endIso);
         },
+        holds(topic, startIso, endIso, stepMs) {
+            return metrics.holdsForTopic(topic, startIso, endIso, stepMs);
+        },
         poll(topic, afterIso, untilIso) {
             return metrics.pollTopic(topic, afterIso, untilIso);
         }
@@ -64,6 +67,13 @@ export function pgMetricsRead(metrics) {
  * @param {string} unit - unit string
  * @returns {object} sensor with name current measurements stream
  */
+async function windowed(read, spec) {
+    const rows = spec.extra === undefined
+        ? await read[spec.method](spec.topic, spec.range.start.toISOString(), spec.range.end.toISOString())
+        : await read[spec.method](spec.topic, spec.range.start.toISOString(), spec.range.end.toISOString(), spec.extra);
+    return seriesRows(rows, spec.unit);
+}
+
 export default function metricsSensor(read, topic, displayName, unit) {
     return {
         name() {
@@ -81,30 +91,17 @@ export default function metricsSensor(read, topic, displayName, unit) {
                 unit
             };
         },
-        async measurements(range, step) {
-            const rows = await read.range(
-                topic,
-                range.start.toISOString(),
-                range.end.toISOString(),
-                step
-            );
-            return seriesRows(rows, unit);
+        measurements(range, step) {
+            return windowed(read, { method: 'range', topic, range, extra: step, unit });
+        },
+        rises(range) {
+            return windowed(read, { method: 'rises', topic, range, extra: undefined, unit });
         },
-        async rises(range) {
-            const rows = await read.rises(
-                topic,
-                range.start.toISOString(),
-                range.end.toISOString()
-            );
-            return seriesRows(rows, unit);
+        changes(range) {
+            return windowed(read, { method: 'changes', topic, range, extra: undefined, unit });
         },
-        async changes(range) {
-            const rows = await read.changes(
-                topic,
-                range.start.toISOString(),
-                range.end.toISOString()
-            );
-            return seriesRows(rows, unit);
+        holds(range, step) {
+            return windowed(read, { method: 'holds', topic, range, extra: step, unit });
         },
         stream(since, step, callback, clock) {
             return pollStream({ read, topic, since, step, callback, clock, unit });

package/src/infrastructure/persistence/pg/holdsSql.js

@@ -0,0 +1,32 @@
+/**
+ * Coverage holds of one Postgres metrics topic.
+ *
+ * $1 topic, $2 start, $3 exclusive end, $4 gap milliseconds.
+ *
+ * @returns {string} SQL
+ */
+export default function holdsSql() {
+    return `WITH windowed AS (
+            SELECT ts, value FROM metrics
+            WHERE topic = $1 AND ts >= $2 AND ts < $3
+        ), lagged AS (
+            SELECT ts, value,
+                   LAG(ts) OVER (ORDER BY ts) AS prev_ts,
+                   LAG(value) OVER (ORDER BY ts) AS prev
+            FROM windowed
+        ), flagged AS (
+            SELECT ts, value,
+                   CASE WHEN prev IS NULL
+                         OR value IS DISTINCT FROM prev
+                         OR (EXTRACT(EPOCH FROM (ts - prev_ts)) * 1000) > $4
+                        THEN 1 ELSE 0 END AS fresh
+            FROM lagged
+        ), grouped AS (
+            SELECT ts, value, SUM(fresh) OVER (ORDER BY ts) AS grp
+            FROM flagged
+        )
+        SELECT MIN(ts) AS ts, MAX(ts) AS until, MIN(value) AS value
+        FROM grouped
+        GROUP BY grp
+        ORDER BY ts`;
+}

package/src/infrastructure/persistence/pg/metrics.js

@@ -1,8 +1,19 @@
 import risesSql from './risesSql.js';
 import changesSql from './changesSql.js';
+import holdsSql from './holdsSql.js';
 import queryTopic from './queryTopic.js';
 import rangeSql from './rangeSql.js';
 
+async function pollRows(pool, topic, afterIso, untilIso) {
+    const result = await pool.query(
+        'SELECT ts, value FROM metrics WHERE topic = $1 AND ts > $2 AND ts <= $3 ORDER BY ts LIMIT 100',
+        [topic, afterIso, untilIso]
+    );
+    return result.rows.map((row) => {
+        return { ts: row.ts, value: row.value };
+    });
+}
+
 export default function metricsStatePg(pool) {
     return {
         async latestForTopic(topic) {
@@ -22,14 +33,16 @@ export default function metricsStatePg(pool) {
         changesForTopic(topic, startIso, endIso) {
             return queryTopic(pool, changesSql(), [topic, startIso, endIso]);
         },
-        async pollTopic(topic, afterIso, untilIso) {
-            const result = await pool.query(
-                'SELECT ts, value FROM metrics WHERE topic = $1 AND ts > $2 AND ts <= $3 ORDER BY ts LIMIT 100',
-                [topic, afterIso, untilIso]
-            );
-            return result.rows.map((row) => {
-                return { ts: row.ts, value: row.value };
-            });
+        holdsForTopic(topic, startIso, endIso, stepMs) {
+            return queryTopic(pool, holdsSql(), [
+                topic,
+                startIso,
+                endIso,
+                Math.max(1, Number(stepMs) || 1000)
+            ]);
+        },
+        pollTopic(topic, afterIso, untilIso) {
+            return pollRows(pool, topic, afterIso, untilIso);
         },
         async insertRows(items) {
             const topics = items.map((row) => {

package/src/infrastructure/persistence/pg/queryTopic.js

@@ -9,6 +9,10 @@
 export default async function queryTopic(pool, sql, params) {
     const result = await pool.query(sql, params);
     return result.rows.map((row) => {
-        return { ts: row.ts, value: Number(row.value) };
+        const item = { ts: row.ts, value: Number(row.value) };
+        if (row.until) {
+            item.until = row.until;
+        }
+        return item;
     });
 }

package/src/infrastructure/persistence/seriesRows.js

@@ -11,11 +11,15 @@ function parseTimestamp(ts) {
  */
 export default function seriesRows(rows, unit) {
     return rows.map((row) => {
-        return {
+        const point = {
             timestamp: parseTimestamp(row.ts),
             value: row.value,
             unit
         };
+        if (row.until) {
+            point.until = parseTimestamp(row.until);
+        }
+        return point;
     });
 }