@yarkivaev/scada 1.5.0 → 2.3.46

package/src/infrastructure/http/plant/streams/operationStream.js

@@ -0,0 +1,42 @@
+import operationJson from '../json/operationJson.js';
+import { route, sseResponse } from '@yarkivaev/simple-server';
+
+/**
+ * Operations SSE route for plant-wide create, update, and delete events.
+ *
+ * @param {string} basePath - base URL path
+ * @param {object} plant - plant domain object
+ * @param {function} clock - time provider
+ * @returns {array} route objects
+ *
+ * @example
+ *   operationStream('/api/v1', plant, clock);
+ */
+export default function operationStream(basePath, plant, clock) {
+    return [
+        route('GET', `${basePath}/operations/stream`, (req, res) => {
+            const sse = sseResponse(res, clock);
+            sse.heartbeat();
+            if (!plant.operations) {
+                sse.close();
+                return;
+            }
+            const subscription = plant.operations.stream((event) => {
+                if (event.type === 'created' && event.operation) {
+                    sse.emit('operation_created', operationJson(event.operation));
+                } else if (event.type === 'updated' && event.operation) {
+                    sse.emit('operation_updated', operationJson(event.operation));
+                } else if (event.type === 'deleted' && event.operation) {
+                    sse.emit('operation_deleted', operationJson(event.operation));
+                }
+            });
+            const heartbeat = setInterval(() => {
+                sse.heartbeat();
+            }, 30000);
+            req.on('close', () => {
+                clearInterval(heartbeat);
+                subscription.cancel();
+            });
+        })
+    ];
+}

package/src/infrastructure/http/plant/streams/timelineStream.js

@@ -0,0 +1,97 @@
+import machineInPlant from '../../../../application/machineInPlant.js';
+import { route, sseResponse } from '@yarkivaev/simple-server';
+
+function segmentPayload(segment) {
+    const start = segment.start_time || segment.startTime;
+    const end = segment.end_time || segment.endTime;
+    const data = {
+        name: segment.name,
+        start: start.toISOString(),
+        end: end.toISOString(),
+        duration: segment.duration
+    };
+    if (segment.options) {
+        data.options = segment.options;
+    }
+    if (segment.tags) {
+        data.tags = segment.tags;
+    }
+    if (segment.properties) {
+        data.properties = segment.properties;
+    }
+    return data;
+}
+
+/**
+ * Timeline SSE routes for segments and label requests.
+ *
+ * @param {string} basePath - base URL path
+ * @param {object} plant - plant domain object
+ * @param {function} clock - time provider
+ * @returns {array} route objects
+ *
+ * @example
+ *   timelineStream('/api/v1', plant, clock);
+ */
+export default function timelineStream(basePath, plant, clock) {
+    return [
+        route('GET', `${basePath}/machines/:machineId/segments/stream`, (req, res, params) => {
+            const sse = sseResponse(res, clock);
+            sse.heartbeat();
+            const result = machineInPlant(plant, params.machineId);
+            if (!result) {
+                sse.close();
+                return;
+            }
+            const subscription = result.machine.timeline.stream((event) => {
+                if (event.type === 'created' && event.segment) {
+                    sse.emit('segment_created', segmentPayload(event.segment));
+                } else if (event.type === 'resolved' && event.segment) {
+                    sse.emit('segment_resolved', segmentPayload(event.segment));
+                }
+            });
+            const heartbeat = setInterval(() => {
+                sse.heartbeat();
+            }, 30000);
+            req.on('close', () => {
+                clearInterval(heartbeat);
+                subscription.cancel();
+            });
+        }),
+        route('GET', `${basePath}/machines/:machineId/requests/stream`, (req, res, params) => {
+            const sse = sseResponse(res, clock);
+            sse.heartbeat();
+            const result = machineInPlant(plant, params.machineId);
+            if (!result) {
+                sse.close();
+                return;
+            }
+            const subscription = result.machine.timeline.stream((event) => {
+                if (event.type === 'created' && event.request) {
+                    const reqItem = event.request;
+                    const start = reqItem.start_time || reqItem.startTime;
+                    const end = reqItem.end_time || reqItem.endTime;
+                    sse.emit('request_created', {
+                        id: reqItem.id,
+                        segment: {
+                            name: reqItem.name,
+                            start: start.toISOString(),
+                            end: end.toISOString(),
+                            duration: reqItem.duration
+                        },
+                        options: reqItem.options
+                    });
+                } else if (event.type === 'resolved' && event.request) {
+                    sse.emit('request_resolved', { id: event.request.id });
+                }
+            });
+            const heartbeat = setInterval(() => {
+                sse.heartbeat();
+            }, 30000);
+            req.on('close', () => {
+                clearInterval(heartbeat);
+                subscription.cancel();
+            });
+        })
+    ];
+}

package/src/infrastructure/http/plant/timelineOperator.js

@@ -0,0 +1,87 @@
+import operatorById from '../../operators/operatorById.js';
+import { errorResponse } from '@yarkivaev/simple-server';
+
+function routeError(code, message, status) {
+    const err = new Error(message);
+    err.routeCode = code;
+    err.routeStatus = status;
+    return err;
+}
+
+/**
+ * Resolves anonymous display name from an optional client marker.
+ *
+ * @param {object} body - parsed JSON body with optional client
+ * @param {object} cfg - timelineOperator options with anonymousUsers and defaultUser
+ * @returns {string} display name for the decision audit row
+ */
+function anonymousName(body, cfg) {
+    const key = body && body.client;
+    const map = cfg.anonymousUsers || {};
+    if (typeof key === 'string' && Object.hasOwn(map, key)) {
+        return map[key];
+    }
+    return cfg.defaultUser || 'hmi-kiosk';
+}
+
+/**
+ * Resolves operator audit context for timeline write routes.
+ *
+ * @param {object} options - provider, requireOperator, defaultUser, anonymousUsers
+ * @returns {object} gate with resolve(body) returning audit context
+ *
+ * @example
+ *   const gate = timelineOperator({
+ *     provider, requireOperator: true, defaultUser: 'hmi-kiosk',
+ *     anonymousUsers: { hmi: 'Anonymous HMI user' }
+ *   });
+ *   const audit = await gate.resolve({ operatorId: 2 });
+ */
+export default function timelineOperator(options) {
+    const cfg = options || {};
+    const lookup = cfg.provider ? operatorById(cfg.provider) : undefined;
+    return {
+        /**
+         * Builds audit context from request body operatorId field.
+         *
+         * @param {object} body - parsed JSON body with optional operatorId
+         * @returns {Promise<object>} audit with id, displayName, decidedAt
+         */
+        async resolve(body) {
+            const { operatorId } = body;
+            const decidedAt = new Date();
+            if (cfg.requireOperator && (operatorId === undefined || operatorId === null)) {
+                throw routeError('FORBIDDEN', 'operator required', 403);
+            }
+            if (operatorId !== undefined && operatorId !== null) {
+                if (!lookup) {
+                    throw routeError('SERVICE_UNAVAILABLE', 'operators catalog unavailable', 503);
+                }
+                const row = await lookup.resolve(operatorId);
+                if (!row) {
+                    throw routeError('BAD_REQUEST', `unknown operator id ${operatorId}`, 400);
+                }
+                return { id: row.id, displayName: row.displayName, decidedAt };
+            }
+            return {
+                id: undefined,
+                displayName: anonymousName(body, cfg),
+                decidedAt
+            };
+        },
+        /**
+         * Sends a route error response when resolve threw a route error.
+         *
+         * @param {object} res - HTTP response
+         * @param {Error} err - error from resolve()
+         * @returns {boolean} true when a response was sent
+         */
+        sendError(res, err) {
+            if (err.routeCode && err.routeStatus) {
+                errorResponse(err.routeCode, err.message, err.routeStatus).send(res);
+                return true;
+            }
+            return false;
+        }
+    };
+}

package/src/infrastructure/ingest/activity/activityTracking.js

@@ -0,0 +1,74 @@
+import {
+    batch,
+    circuit,
+    clickhouseSink,
+    clock,
+    lokiSource,
+    timedBatch
+} from '@yarkivaev/source-to-sink';
+import activityCodec from './activityTransformer.js';
+
+/**
+ * Pipeline for streaming Loki HTTP access logs to ClickHouse.
+ *
+ * Polls Loki for log entries matching the query and batches them
+ * before inserting into ClickHouse `activity.http_access` table.
+ * Uses circuit breaker for failure isolation.
+ *
+ * @example
+ *   const pipeline = activityTracking(
+ *     'http://localhost:3100',
+ *     'http://localhost:8123',
+ *     '{app="traefik"}',
+ *     { poll: 10, size: 100, interval: 5, threshold: 5, timeout: 60 }
+ *   );
+ *   pipeline.start();
+ *   // ... later
+ *   pipeline.stop();
+ *
+ * @param {string} loki - Loki base URL
+ * @param {string} clickhouse - ClickHouse URL
+ * @param {string} query - LogQL query string
+ * @param {object} config - Pipeline configuration
+ * @param {number} config.poll - Loki polling interval in seconds
+ * @param {number} config.size - Batch size before flush
+ * @param {number} config.interval - Seconds before time-based flush
+ * @param {number} config.threshold - Circuit breaker failure threshold
+ * @param {number} config.timeout - Circuit breaker timeout in seconds
+ * @returns {object} Pipeline with start() and stop() methods
+ */
+export default function activityTracking(loki, clickhouse, query, config) {
+    if (typeof loki !== 'string' || loki.length === 0) {
+        throw new Error('Loki URL must be a non-empty string');
+    }
+    if (typeof clickhouse !== 'string' || clickhouse.length === 0) {
+        throw new Error('ClickHouse URL must be a non-empty string');
+    }
+    if (typeof query !== 'string' || query.length === 0) {
+        throw new Error('Query must be a non-empty string');
+    }
+    if (!config || typeof config !== 'object') {
+        throw new Error('Config must be an object');
+    }
+    const clk = clock();
+    const breaker = circuit(config.threshold, config.timeout, clk);
+    const sink = clickhouseSink(clickhouse, 'activity.http_access');
+    const collector = timedBatch(batch(sink, config.size, breaker), config.interval);
+    const transformer = activityCodec(collector);
+    const source = lokiSource(loki, query, config.poll, transformer, clk);
+    return {
+        /**
+         * Starts the pipeline.
+         */
+        start() {
+            source.start();
+        },
+        /**
+         * Stops the pipeline.
+         */
+        stop() {
+            source.stop();
+            collector.stop();
+        }
+    };
+}

package/src/infrastructure/ingest/activity/activityTransformer.js

@@ -0,0 +1,45 @@
+/**
+ * Transformer for converting Traefik JSON log entries to activity records.
+ *
+ * Parses Traefik access log format and extracts user activity fields.
+ * Skips entries that cannot be parsed or lack required fields.
+ *
+ * @example
+ *   const transformer = activityTransformer(collector);
+ *   transformer.accept({ ts: 123, line: '{"StartUTC":"...","RequestMethod":"GET",...}' });
+ *
+ * @param {object} collector - Collector with accept() method
+ * @returns {object} Transformer with accept() method
+ */
+export default function activityCodec(collector) {
+    if (!collector || typeof collector.accept !== 'function') {
+        throw new Error('Collector must have an accept() method');
+    }
+    return {
+        /**
+         * Accepts Loki log entry and transforms to activity record.
+         *
+         * @param {object} entry - Log entry with ts and line
+         */
+        accept(entry) {
+            try {
+                const log = JSON.parse(entry.line);
+                if (!log.StartUTC || !log.RequestMethod || !log.RequestPath) {
+                    return;
+                }
+                collector.accept({
+                    ts: new Date(log.StartUTC).getTime(),
+                    user: log['request_Remote-User'] || '',
+                    email: log['request_Remote-Email'] || '',
+                    method: log.RequestMethod || '',
+                    path: log.RequestPath || '',
+                    status: parseInt(log.DownstreamStatus, 10) || 0,
+                    duration: parseFloat(log.Duration) / 1e9 || 0,
+                    ip: log.ClientHost || ''
+                });
+            } catch {
+                // Skip malformed log entries
+            }
+        }
+    };
+}

package/src/infrastructure/ingest/codecs/alertCodec.js

@@ -0,0 +1,56 @@
+import processingErrorLog from '../processingErrorLog.js';
+
+/**
+ * Codec for converting raw STOMP alert messages to PostgreSQL-ready records.
+ *
+ * Parses JSON payloads and extracts name, machine, severity, status,
+ * and timestamps. Translates rule names to human-readable messages
+ * using the provided translation map. Converts epoch timestamps to
+ * ISO strings for PostgreSQL TIMESTAMPTZ columns.
+ *
+ * @example
+ *   const codec = alertCodec(collector, { low_cosphi: 'Switch off compensation' });
+ *   codec.accept({ destination: '/exchange/scada.alerts', payload: '{"name":"low_cosphi","machine":"m2","severity":"warning","status":"pending","start":1700000000}' });
+ *
+ * @param {object} collector - Collector with accept() method
+ * @param {object} translations - Map of rule names to human-readable messages
+ * @returns {object} Codec with accept() method
+ */
+export default function alertCodec(collector, translations) {
+    if (!collector || typeof collector.accept !== 'function') {
+        throw new Error('Collector must have an accept() method');
+    }
+    return {
+        /**
+         * Accepts a raw STOMP message and forwards the decoded record to the collector.
+         *
+         * @param {object} raw - Raw message with destination and payload
+         */
+        async accept(raw) {
+            try {
+                const parsed = JSON.parse(raw.payload);
+                if (typeof parsed.name !== 'string') {
+                    throw new Error('Alert missing name field');
+                }
+                if (typeof parsed.machine !== 'string') {
+                    throw new Error('Alert missing machine field');
+                }
+                const timestamp = new Date(parsed.start);
+                if (isNaN(timestamp.getTime())) {
+                    throw new RangeError(`Invalid epoch timestamp: ${parsed.start}`);
+                }
+                await collector.accept({
+                    name: parsed.name,
+                    message: translations[parsed.name] || parsed.name,
+                    machine: parsed.machine,
+                    severity: parsed.severity,
+                    status: parsed.status,
+                    timestamp: timestamp.toISOString()
+                });
+            } catch (error) {
+                processingErrorLog('alert_codec', error, { destination: raw.destination, payload: raw.payload });
+                throw error;
+            }
+        }
+    };
+}

package/src/infrastructure/ingest/codecs/segmentCodec.js

@@ -0,0 +1,30 @@
+import segmentNormalize from '../../../domain/segment/normalize.js';
+import processingErrorLog from '../processingErrorLog.js';
+
+/**
+ * Codec for converting raw STOMP segment messages to PostgreSQL-ready records.
+ *
+ * @param {object} collector - Collector with accept() method
+ * @returns {object} Codec with accept() method
+ *
+ * @example
+ *   const codec = segmentCodec(collector);
+ *   codec.accept({ destination: '/exchange/scada.segments', payload: '{"machine":"m2","name":"on","start":1700000000,"end":1700003600,"duration":3600}' });
+ */
+export default function segmentCodec(collector) {
+    if (!collector || typeof collector.accept !== 'function') {
+        throw new Error('Collector must have an accept() method');
+    }
+    return {
+        async accept(raw) {
+            try {
+                const parsed = JSON.parse(raw.payload);
+                const record = segmentNormalize(parsed);
+                await collector.accept(record);
+            } catch (error) {
+                processingErrorLog('segment_codec', error, { destination: raw.destination, payload: raw.payload });
+                throw error;
+            }
+        }
+    };
+}

package/src/infrastructure/ingest/codecs/userDecisionCodec.js

@@ -0,0 +1,78 @@
+import processingErrorLog from '../processingErrorLog.js';
+
+function parseStartTime(parsed) {
+    const startTime = new Date(parsed.start * 1000);
+    if (isNaN(startTime.getTime())) {
+        throw new RangeError(`Invalid epoch timestamp: ${parsed.start}`);
+    }
+    return startTime;
+}
+
+function parseDecidedAt(parsed) {
+    if (parsed.decided_at === undefined || parsed.decided_at === null) {
+        return new Date().toISOString();
+    }
+    const decided = new Date(parsed.decided_at * 1000);
+    if (isNaN(decided.getTime())) {
+        throw new RangeError(`Invalid decided_at timestamp: ${parsed.decided_at}`);
+    }
+    return decided.toISOString();
+}
+
+function decisionRecord(raw, parsed, startTime, decidedAt) {
+    const record = {
+        machine: parsed.machine,
+        startTime: startTime.toISOString(),
+        username: parsed.user,
+        payload: raw.payload,
+        decidedAt
+    };
+    if (parsed.operator_id !== undefined && parsed.operator_id !== null) {
+        record.operatorId = Number(parsed.operator_id);
+    }
+    return record;
+}
+
+/**
+ * Codec for converting raw STOMP user decision messages to sink-ready records.
+ *
+ * Parses JSON payloads and extracts machine, start timestamp, user, operator_id,
+ * decided_at, and the full raw payload for audit storage.
+ *
+ * @example
+ *   const codec = userDecisionCodec(collector);
+ *   codec.accept({ destination: '/exchange/scada.user_decisions',
+ *                  payload: '{"machine":"m2","start":1700000000,"user":"op1","operator_id":2}' });
+ *
+ * @param {object} collector - Collector with accept() method
+ * @returns {object} Codec with accept() method
+ */
+export default function userDecisionCodec(collector) {
+    if (!collector || typeof collector.accept !== 'function') {
+        throw new Error('Collector must have an accept() method');
+    }
+    return {
+        /**
+         * Accepts a raw STOMP message and forwards the decoded record to the collector.
+         *
+         * @param {object} raw - Raw message with destination and payload
+         */
+        async accept(raw) {
+            try {
+                const parsed = JSON.parse(raw.payload);
+                if (typeof parsed.machine !== 'string') {
+                    throw new Error('Decision missing machine field');
+                }
+                if (typeof parsed.user !== 'string') {
+                    throw new Error('Decision missing user field');
+                }
+                const startTime = parseStartTime(parsed);
+                const decidedAt = parseDecidedAt(parsed);
+                await collector.accept(decisionRecord(raw, parsed, startTime, decidedAt));
+            } catch (error) {
+                processingErrorLog('decision_codec', error, { destination: raw.destination, payload: raw.payload });
+                throw error;
+            }
+        }
+    };
+}

package/src/infrastructure/ingest/cooldown.js

@@ -0,0 +1,24 @@
+/**
+ * Decorator that rate-limits an issue callback by message.
+ * Blocks duplicate messages within the cooldown interval.
+ *
+ * @param {function} issue - callback to wrap (message, timestamp)
+ * @param {number} interval - cooldown interval in milliseconds
+ * @returns {function} wrapped callback that enforces cooldown
+ *
+ * @example
+ *   const limited = cooldown(console.log, 60000);
+ *   limited('alert', new Date()); // fires
+ *   limited('alert', new Date()); // blocked (within 60s)
+ */
+export default function cooldown(issue, interval) {
+    const history = {};
+    return function limited(message, timestamp) {
+        const now = timestamp.getTime();
+        const last = history[message] || 0;
+        if (now - last >= interval) {
+            history[message] = now;
+            issue(message, timestamp);
+        }
+    };
+}

package/src/infrastructure/ingest/db/migrate.js

@@ -0,0 +1,78 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { migrationApplies, migrationSinkRunnable } from './migrationProfile.js';
+/* eslint-disable no-await-in-loop */
+
+function migrationSelected(name, profile, privileged) {
+    if (!name.endsWith('.sql') || !migrationApplies(name, profile)) {
+        return false;
+    }
+    const sinkRunnable = migrationSinkRunnable(name);
+    return privileged ? !sinkRunnable : sinkRunnable;
+}
+
+async function ensureMigrationsTable(pool) {
+    await pool.query(`
+        CREATE TABLE IF NOT EXISTS schema_migrations (
+            version TEXT PRIMARY KEY,
+            applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
+        )
+    `);
+}
+
+async function applyMigrationFiles(pool, migrationsDir, profile, privileged) {
+    const files = (await fs.readdir(migrationsDir))
+        .filter((name) => {
+            return migrationSelected(name, profile, privileged);
+        })
+        .sort((left, right) => {
+            return left.localeCompare(right);
+        });
+    for (const file of files) {
+        const { rowCount } = await pool.query(
+            'SELECT 1 FROM schema_migrations WHERE version = $1',
+            [file]
+        );
+        if (rowCount === 0) {
+            const sql = await fs.readFile(path.join(migrationsDir, file), 'utf8');
+            await pool.query('BEGIN');
+            try {
+                await pool.query(sql);
+                await pool.query(
+                    'INSERT INTO schema_migrations (version) VALUES ($1)',
+                    [file]
+                );
+                await pool.query('COMMIT');
+            } catch (error) {
+                await pool.query('ROLLBACK');
+                throw error;
+            }
+        }
+    }
+}
+
+/**
+ * Applies SQL migrations runnable as supervisor_sink.
+ *
+ * @param {object} pool - pg.Pool instance.
+ * @param {string} migrationsDir - absolute path to migration files.
+ * @param {string} [profile] - central or edge; filters E/C prefixed files
+ * @returns {Promise<void>}
+ */
+export default async function migrate(pool, migrationsDir, profile = 'edge') {
+    await ensureMigrationsTable(pool);
+    await applyMigrationFiles(pool, migrationsDir, profile, false);
+}
+
+/**
+ * Applies privilege migrations that require Postgres superuser (scada).
+ *
+ * @param {object} pool - pg.Pool connected as scada or other superuser.
+ * @param {string} migrationsDir - absolute path to migration files.
+ * @param {string} [profile] - central or edge; filters E/C prefixed files
+ * @returns {Promise<void>}
+ */
+export async function migratePrivileged(pool, migrationsDir, profile = 'central') {
+    await ensureMigrationsTable(pool);
+    await applyMigrationFiles(pool, migrationsDir, profile, true);
+}

package/src/infrastructure/ingest/db/migrationProfile.js

@@ -0,0 +1,35 @@
+/**
+ * Returns whether a migration file applies to the given database profile.
+ *
+ * @param {string} fileName - migration file name
+ * @param {string} profile - sink database profile: central or edge
+ * @returns {boolean}
+ */
+export function migrationApplies(fileName, profile) {
+    if (fileName.startsWith('E')) {
+        return profile === 'edge';
+    }
+    if (fileName.startsWith('C')) {
+        return profile === 'central';
+    }
+    return true;
+}
+
+const privilegedOnly = new Set([
+    'C0002__central_revoke_delete.sql',
+    'C0004__central_operations_grants.sql',
+    'C0005__central_operators_grants.sql',
+    'C0006__central_operators_registration.sql',
+    'E0002__edge_retention_delete.sql',
+    'E0003__edge_operations_grants.sql'
+]);
+
+/**
+ * Privilege migrations run as scada only, not supervisor_sink.
+ *
+ * @param {string} fileName - migration file name
+ * @returns {boolean}
+ */
+export function migrationSinkRunnable(fileName) {
+    return !privilegedOnly.has(fileName);
+}