@yarkivaev/scada 1.4.0 → 2.3.45

package/src/{clickhouseSensor.js → infrastructure/persistence/clickhouse/sensor.js}

@@ -4,7 +4,8 @@
  * Reads sensor measurements from scada.metrics table
  * and provides real-time streaming via polling.
  * Supports time-based downsampling using ClickHouse aggregation.
- * Returns value of 0 for missing data to comply with no-null-return principle.
+ * Live streams share a connection-level batch hub so many sensors
+ * cost one ClickHouse query per tick instead of one query per sensor.
  *
  * @param {object} connection - ClickHouse connection with query method
  * @param {string} topic - Metric topic in format '{machine}/{sensor}'
@@ -13,17 +14,18 @@
  * @returns {object} sensor with name, current, measurements and stream methods
  *
  * @example
- *   const sensor = clickhouseSensor(conn, 'icht1/voltage', 'Voltage', 'V');
+ *   const sensor = clickhouseSensor(conn, 'm1/voltage', 'Voltage', 'V');
  *   sensor.name(); // 'Voltage'
  *   await sensor.current(); // { found: true, timestamp, value, unit } or { found: false }
  *   await sensor.measurements({ start, end }, 60000); // downsampled to 1-minute intervals
  *   sensor.stream(since, 1000, callback); // live stream
  */
+import clickhouseStreamHub from './streamHub.js';
+
 function formatDateTime(date) {
     return date.toISOString().replace('Z', '').replace('T', ' ');
 }
 
-// eslint-disable-next-line max-lines-per-function
 export default function clickhouseSensor(connection, topic, displayName, unit) {
     return {
         name() {
@@ -64,30 +66,10 @@ export default function clickhouseSensor(connection, topic, displayName, unit) {
             });
         },
         stream(since, step, callback, clock) {
-            const time = clock || (() => { return new Date(); });
-            let lastTs = since;
-            const timer = setInterval(async () => {
-                try {
-                    const rows = await connection.query(
-                        `SELECT ts, value FROM scada.metrics
-                         WHERE topic = {topic:String} AND ts > {since:DateTime64(3)} AND ts <= {until:DateTime64(3)}
-                         ORDER BY ts LIMIT 100`,
-                        { topic, since: formatDateTime(lastTs), until: formatDateTime(time()) }
-                    );
-                    rows.forEach((row) => {
-                        const timestamp = new Date(`${row.ts}Z`);
-                        callback({ timestamp, value: row.value, unit });
-                        lastTs = timestamp;
-                    });
-                } catch (err) {
-                    console.error(`Stream query failed for ${topic} on ${connection.url()}:`, err.message); // eslint-disable-line no-console
-                }
-            }, step);
-            return {
-                cancel() {
-                    clearInterval(timer);
-                }
-            };
+            return clickhouseStreamHub(connection).watch(topic, since, step, callback, {
+                unit,
+                clock
+            });
         }
     };
 }

package/src/infrastructure/persistence/clickhouse/streamHub.js

@@ -0,0 +1,165 @@
+/**
+ * Shared ClickHouse stream hub.
+ *
+ * Coalesces many topic watchers on one connection into one batched poll per
+ * step interval, cutting ClickHouse CPU from O(sensors) to O(pulses).
+ *
+ * @example
+ *   const hub = clickhouseStreamHub(connection);
+ *   const sub = hub.watch('m1/V', since, 1000, onRow, { unit: 'V', clock });
+ *   sub.cancel();
+ */
+import pollTopicCursors from './pollTopicCursors.js';
+
+const hubs = new WeakMap();
+
+function defaultSchedule(step, fn) {
+    const timer = setInterval(fn, step);
+    return () => {
+        clearInterval(timer);
+    };
+}
+
+function parseTimestamp(ts) {
+    return new Date(`${ts}Z`);
+}
+
+function bucketKey(step) {
+    return String(step);
+}
+
+function createBucket(hub, step, clock) {
+    return {
+        step,
+        clock,
+        watchers: new Map(),
+        stop: hub.schedule(step, () => {
+            return hub.pulse(step);
+        })
+    };
+}
+
+function addWatcher(bucket, topic, watcher) {
+    if (!bucket.watchers.has(topic)) {
+        bucket.watchers.set(topic, new Set());
+    }
+    bucket.watchers.get(topic).add(watcher);
+}
+
+function removeWatcher(bucket, topic, watcher) {
+    const group = bucket.watchers.get(topic);
+    if (!group) {
+        return;
+    }
+    group.delete(watcher);
+    if (group.size === 0) {
+        bucket.watchers.delete(topic);
+    }
+}
+
+function cursorsOf(bucket) {
+    const oldest = new Map();
+    bucket.watchers.forEach((group, topic) => {
+        group.forEach((watcher) => {
+            const prev = oldest.get(topic);
+            if (!prev || watcher.lastTs.getTime() < prev.getTime()) {
+                oldest.set(topic, watcher.lastTs);
+            }
+        });
+    });
+    const cursors = [];
+    oldest.forEach((since, topic) => {
+        cursors.push({ topic, since });
+    });
+    return cursors;
+}
+
+function dispatch(bucket, rows) {
+    rows.forEach((row) => {
+        const group = bucket.watchers.get(row.topic);
+        if (!group) {
+            return;
+        }
+        const timestamp = parseTimestamp(row.ts);
+        group.forEach((watcher) => {
+            if (timestamp.getTime() <= watcher.lastTs.getTime()) {
+                return;
+            }
+            watcher.callback({ timestamp, value: row.value, unit: watcher.unit });
+            watcher.lastTs = timestamp;
+        });
+    });
+}
+
+function buildHub(connection, schedule) {
+    const buckets = new Map();
+    const hub = {
+        schedule,
+        async pulse(step) {
+            const bucket = buckets.get(bucketKey(step));
+            if (!bucket || bucket.watchers.size === 0) {
+                return;
+            }
+            try {
+                const rows = await pollTopicCursors(
+                    connection,
+                    cursorsOf(bucket),
+                    bucket.clock()
+                );
+                dispatch(bucket, rows);
+            } catch (err) {
+                console.error( // eslint-disable-line no-console
+                    `Stream batch failed on ${connection.url()}:`,
+                    err.message
+                );
+            }
+        },
+        watch(topic, since, step, callback, options) {
+            const key = bucketKey(step);
+            if (!buckets.has(key)) {
+                buckets.set(key, createBucket(hub, step, options.clock || (() => {
+                    return new Date();
+                })));
+            }
+            const bucket = buckets.get(key);
+            const watcher = { lastTs: since, callback, unit: options.unit };
+            addWatcher(bucket, topic, watcher);
+            return {
+                cancel() {
+                    removeWatcher(bucket, topic, watcher);
+                    if (bucket.watchers.size === 0) {
+                        bucket.stop();
+                        buckets.delete(key);
+                    }
+                }
+            };
+        }
+    };
+    return hub;
+}
+
+/**
+ * Creates a stream hub for connection, replacing any previous hub.
+ *
+ * @param {object} connection - shared ClickHouse connection
+ * @param {function} [schedule] - (step, fn) => cancel, for tests
+ * @returns {object} hub with watch(topic, since, step, callback, options)
+ */
+export function createClickhouseStreamHub(connection, schedule) {
+    const hub = buildHub(connection, schedule || defaultSchedule);
+    hubs.set(connection, hub);
+    return hub;
+}
+
+/**
+ * Returns the shared hub for connection, creating one if needed.
+ *
+ * @param {object} connection - shared ClickHouse connection
+ * @returns {object} hub with watch(topic, since, step, callback, options)
+ */
+export default function clickhouseStreamHub(connection) {
+    if (!hubs.has(connection)) {
+        return createClickhouseStreamHub(connection, defaultSchedule);
+    }
+    return hubs.get(connection);
+}

package/src/infrastructure/persistence/memory/alerts.js

@@ -0,0 +1,21 @@
+/**
+ * In-memory alert hydration port for tests and local runs.
+ *
+ * @param {object} store - shared mutable store with alerts array
+ * @returns {object} alerts port with listUnacknowledged
+ */
+export default function alertsStateMemory(store) {
+    return {
+        listUnacknowledged(filters) {
+            let rows = store.alerts.filter((row) => {
+                return row.acknowledged === false;
+            });
+            if (filters.machine) {
+                rows = rows.filter((row) => {
+                    return row.machine === filters.machine;
+                });
+            }
+            return rows;
+        }
+    };
+}

package/src/infrastructure/persistence/memory/checkpoints.js

@@ -0,0 +1,98 @@
+function parseJsonField(raw) {
+    if (!raw) {
+        return null;
+    }
+    return JSON.parse(raw);
+}
+
+function segmentItem(row) {
+    if (!row) {
+        return null;
+    }
+    return {
+        machine: row.machine,
+        name: row.name,
+        start: new Date(row.start_time).getTime() / 1000,
+        end: new Date(row.end_time).getTime() / 1000,
+        duration: row.duration,
+        tags: parseJsonField(row.tags),
+        options: parseJsonField(row.options),
+        properties: parseJsonField(row.properties)
+    };
+}
+
+function segmentAt(store, machineId, startEpoch) {
+    const startMs = startEpoch * 1000;
+    const row = store.segments.find((item) => {
+        return item.machine === machineId && new Date(item.start_time).getTime() === startMs;
+    });
+    return segmentItem(row);
+}
+
+function replayCursorFor(store, machineId) {
+    const closed = store.segments.filter((row) => {
+        return row.machine === machineId && Number(row.duration) > 0;
+    }).map((row) => {
+        return new Date(row.end_time).getTime() / 1000;
+    });
+    const pending = store.segments.filter((row) => {
+        return row.machine === machineId && Number(row.duration) === 0;
+    }).map((row) => {
+        return new Date(row.start_time).getTime() / 1000;
+    });
+    const closedMax = closed.length ? Math.max(...closed) : null;
+    const pendingMax = pending.length ? Math.max(...pending) : null;
+    let cursor = 0;
+    if (closedMax !== null && pendingMax !== null) {
+        cursor = Math.min(closedMax, pendingMax);
+    } else if (pendingMax !== null) {
+        cursor = pendingMax;
+    } else if (closedMax !== null) {
+        cursor = closedMax;
+    }
+    return cursor;
+}
+
+/**
+ * In-memory checkpoint state port for tests and local runs.
+ *
+ * @param {object} store - shared mutable store with segments and metrics arrays
+ * @returns {object} checkpoints port matching checkpointStatePg shape
+ */
+export default function checkpointStateMemory(store) {
+    return {
+        replayCursor(machineId) {
+            return replayCursorFor(store, machineId);
+        },
+        pendingSegments() {
+            return store.segments.filter((row) => {
+                return Number(row.duration) === 0;
+            }).map((row) => {
+                return {
+                    machine: row.machine,
+                    name: row.name,
+                    start: new Date(row.start_time).getTime() / 1000,
+                    end: new Date(row.end_time).getTime() / 1000
+                };
+            });
+        },
+        readings(topics, fromEpoch) {
+            const fromMs = Number(fromEpoch) * 1000;
+            return store.metrics.filter((row) => {
+                const tsMs = new Date(row.ts).getTime();
+                return topics.includes(row.topic) && tsMs > fromMs;
+            }).sort((a, b) => {
+                return new Date(a.ts) - new Date(b.ts);
+            }).map((row) => {
+                return {
+                    topic: row.topic,
+                    timestamp: new Date(row.ts).getTime() / 1000,
+                    value: Number(row.value)
+                };
+            });
+        },
+        segment(machineId, startEpoch) {
+            return segmentAt(store, machineId, startEpoch);
+        }
+    };
+}

package/src/infrastructure/persistence/memory/metrics.js

@@ -0,0 +1,69 @@
+function bucketKey(tsMs, originMs, stepMs) {
+    const slot = Math.floor((tsMs - originMs) / stepMs);
+    return originMs + slot * stepMs;
+}
+
+function rowsForRange(store, topic, startMs, endMs, stepMs) {
+    const hits = store.metrics.filter((row) => {
+        const tsMs = new Date(row.ts).getTime();
+        return row.topic === topic && tsMs >= startMs && tsMs <= endMs;
+    }).sort((a, b) => {
+        return new Date(a.ts) - new Date(b.ts);
+    });
+    const byBucket = new Map();
+    hits.forEach((row) => {
+        const tsMs = new Date(row.ts).getTime();
+        const key = bucketKey(tsMs, startMs, stepMs);
+        byBucket.set(key, row);
+    });
+    return Array.from(byBucket.entries()).sort((a, b) => {
+        return a[0] - b[0];
+    }).map((entry) => {
+        return { ts: new Date(entry[0]), value: entry[1].value };
+    });
+}
+
+/**
+ * In-memory metrics state port for tests and local runs.
+ *
+ * @param {object} store - shared mutable store with metrics array
+ * @returns {object} metrics port matching metricsStatePg shape
+ */
+export default function metricsStateMemory(store) {
+    return {
+        latestForTopic(topic) {
+            const hits = store.metrics.filter((row) => {
+                return row.topic === topic;
+            }).sort((a, b) => {
+                return new Date(b.ts) - new Date(a.ts);
+            });
+            return hits[0] ?? null;
+        },
+        rangeForTopic(topic, startIso, endIso, stepMs) {
+            const start = new Date(startIso).getTime();
+            const end = new Date(endIso).getTime();
+            const stepSec = Math.max(1, Math.floor(stepMs / 1000));
+            return rowsForRange(store, topic, start, end, stepSec * 1000);
+        },
+        pollTopic(topic, afterIso, untilIso) {
+            const after = new Date(afterIso).getTime();
+            const until = new Date(untilIso).getTime();
+            return store.metrics.filter((row) => {
+                const tsMs = new Date(row.ts).getTime();
+                return row.topic === topic && tsMs > after && tsMs <= until;
+            }).sort((a, b) => {
+                return new Date(a.ts) - new Date(b.ts);
+            }).slice(0, 100).map((row) => {
+                return { ts: row.ts, value: row.value };
+            });
+        },
+        insertRows(items) {
+            let i = 0;
+            while (i < items.length) {
+                const row = items[i];
+                store.metrics.push({ topic: row.topic, ts: new Date(row.ts), value: row.value });
+                i += 1;
+            }
+        }
+    };
+}

package/src/infrastructure/persistence/memory/metricsDisabled.js

@@ -0,0 +1,19 @@
+/**
+ * Metrics port used when SINK_DB_PROFILE is central (ClickHouse owns metrics).
+ */
+export default function metricsStateDisabled() {
+    return {
+        latestForTopic() {
+            return Promise.resolve(null);
+        },
+        rangeForTopic() {
+            return Promise.resolve([]);
+        },
+        pollTopic() {
+            return Promise.resolve([]);
+        },
+        insertRows() {
+            return Promise.reject(new Error('metrics storage is disabled on central profile'));
+        }
+    };
+}

package/src/infrastructure/persistence/memory/operations.js

@@ -0,0 +1,87 @@
+function sameKey(left, right) {
+    return left.key === right.key;
+}
+
+function findRow(store, key) {
+    return store.operations.find((row) => {
+        return sameKey(row, { key });
+    });
+}
+
+function findScoped(store, machineId, key) {
+    return store.operations.find((row) => {
+        return row.machine === machineId && row.key === key;
+    });
+}
+
+function missing(machineId, key) {
+    return new Error(`operation '${key}' not found for machine '${machineId}'`);
+}
+
+function filterList(store, machineId, kind, range) {
+    const from = range.from ?? null;
+    const to = range.to ?? null;
+    return store.operations.filter((row) => {
+        if (row.machine !== machineId || row.kind !== kind) {
+            return false;
+        }
+        if (from && new Date(row.occurred_at).getTime() < new Date(from).getTime()) {
+            return false;
+        }
+        if (to && new Date(row.occurred_at).getTime() > new Date(to).getTime()) {
+            return false;
+        }
+        return true;
+    }).sort((left, right) => {
+        return new Date(left.occurred_at) - new Date(right.occurred_at);
+    });
+}
+
+/**
+ * In-memory operations state port for tests and local runs.
+ *
+ * @param {object} store - shared mutable store with operations array
+ * @returns {object} operations port matching operationStatePg shape
+ */
+export default function operationStateMemory(store) {
+    if (!store.operations) {
+        store.operations = [];
+    }
+    return {
+        upsert(item) {
+            const row = findRow(store, item.key);
+            if (row) {
+                row.machine = item.machine;
+                row.occurred_at = item.occurred_at;
+                row.kind = item.kind;
+                row.payload = item.payload;
+                if (item.source_updated_at) {
+                    row.source_updated_at = item.source_updated_at;
+                }
+                return Promise.resolve({ created: false });
+            }
+            store.operations.push({ ...item });
+            return Promise.resolve({ created: true });
+        },
+        get(machineId, key) {
+            const row = findScoped(store, machineId, key);
+            if (!row) {
+                return Promise.reject(missing(machineId, key));
+            }
+            return Promise.resolve(row);
+        },
+        remove(machineId, key) {
+            const index = store.operations.findIndex((row) => {
+                return row.machine === machineId && row.key === key;
+            });
+            if (index < 0) {
+                return Promise.reject(missing(machineId, key));
+            }
+            const [row] = store.operations.splice(index, 1);
+            return Promise.resolve(row);
+        },
+        listForMachine(machineId, kind, range) {
+            return Promise.resolve(filterList(store, machineId, kind, range));
+        }
+    };
+}

package/src/infrastructure/persistence/memory/segments.js

@@ -0,0 +1,83 @@
+function sameInstant(a, b) {
+    return new Date(a).getTime() === new Date(b).getTime();
+}
+
+function findRow(store, machineId, start) {
+    return store.segments.find((row) => {
+        return row.machine === machineId && sameInstant(row.start_time, start);
+    });
+}
+
+function parseListFilters(range) {
+    const from = range.from ?? null;
+    const to = range.to ?? null;
+    return { from, to };
+}
+
+function filterList(store, machineId, range) {
+    const { from, to } = parseListFilters(range);
+    return store.segments.filter((row) => {
+        if (row.machine !== machineId) {
+            return false;
+        }
+        if (from && new Date(row.end_time).getTime() < new Date(from).getTime() && row.duration !== 0) {
+            return false;
+        }
+        if (to && new Date(row.start_time).getTime() > new Date(to).getTime()) {
+            return false;
+        }
+        return true;
+    }).sort((a, b) => {
+        return new Date(a.start_time) - new Date(b.start_time);
+    });
+}
+
+/**
+ * In-memory segment state port for tests and local runs.
+ *
+ * @param {object} store - shared mutable store with segments array
+ * @returns {object} segments port matching segmentStatePg shape
+ */
+export default function segmentStateMemory(store) {
+    return {
+        listForMachine(machineId, range) {
+            return filterList(store, machineId, range);
+        },
+        rowAt(machineId, start) {
+            const row = findRow(store, machineId, start);
+            return row ?? null;
+        },
+        pendingRequestsForMachine(machineId) {
+            return store.segments.filter((row) => {
+                return row.machine === machineId && row.resolved === false;
+            }).sort((a, b) => {
+                return new Date(a.start_time) - new Date(b.start_time);
+            }).map((row) => {
+                return {
+                    id: row.start_time,
+                    name: row.name,
+                    start_time: row.start_time,
+                    end_time: row.end_time,
+                    duration: row.duration,
+                    options: row.options
+                };
+            });
+        },
+        retag(machineId, start, tagsJson, propertiesJson) {
+            const row = findRow(store, machineId, start);
+            if (row) {
+                row.tags = tagsJson;
+                row.properties = propertiesJson;
+            }
+        },
+        resolveRequest(machineId, startKey, tagsJson, propertiesJson) {
+            const row = findRow(store, machineId, startKey);
+            if (row) {
+                row.tags = tagsJson;
+                row.properties = propertiesJson;
+                row.resolved = true;
+                row.consumed = false;
+            }
+        }
+    };
+}

package/src/infrastructure/persistence/memory/timeline.js

@@ -0,0 +1,56 @@
+import timeline from '../../../domain/timeline/timeline.js';
+
+/**
+ * In-memory segment store with timeline read port.
+ *
+ * @returns {object} items, pending arrays and read port
+ *
+ * @example
+ *   const store = memoryTimelineStore();
+ *   store.items.push({ name: 'on', start_time: new Date(), end_time: new Date(), duration: 0 });
+ */
+export default function memoryTimelineStore() {
+    const items = [];
+    const pending = [];
+    return {
+        items,
+        pending,
+        read: {
+            list(range) {
+                if (!range) {
+                    return items.slice();
+                }
+                return items.filter((item) => {
+                    if (range.from && item.end_time < new Date(range.from)) {
+                        return false;
+                    }
+                    if (range.to && item.start_time > new Date(range.to)) {
+                        return false;
+                    }
+                    return true;
+                });
+            },
+            rowAt(start) {
+                return items.find((item) => {
+                    return item.start_time.getTime() === start.getTime();
+                }) || null;
+            },
+            pending() {
+                return pending.filter((item) => {
+                    return !item.resolved;
+                });
+            }
+        }
+    };
+}
+
+/**
+ * Builds in-memory timeline read port wired to a pubsub bus.
+ *
+ * @param {object} store - memoryTimelineStore instance
+ * @param {object} bus - pubsub instance
+ * @returns {object} timeline with list, rowAt, pending, stream
+ */
+export function memoryTimelineRead(store, bus) {
+    return timeline(store.read, bus);
+}