@yarkivaev/scada 2.3.60 → 2.3.61

package/README.md

@@ -91,6 +91,7 @@ Main entry (`import { … } from '@yarkivaev/scada'`):
 | `timeline`, `alerts`, `alert`, `acknowledgedAlert` | Timeline and alerting |
 | `plantApi`, `plantServer`, `siteServer` | HTTP composition |
 | `exportQuery`, `exportStream`, `exportSink`, `exportJob` | Generic export ports over plantApi / `@yarkivaev/scada/client` |
+| `siteSync`, `siteSyncSites`, `siteSyncBind` | Pull kinds from an allowlisted remote plantApi into local persistence |
 | `metricsPlant`, `shopWithTimeline`, `machineInPlant` | Plant wiring helpers |
 | `supervisorSink`, `readDeploymentConfig` | STOMP ingest + PG persistence |
 | `edgeApi`, `stateHttpClient`, `metricsSensor` | Edge HTTP read/write |
@@ -125,6 +126,19 @@ const job = exportJob({ query, transform: (rows) => rows, sink });
 await job.run({ kind: 'segments', machine: 'furnace-α', from, to });
 ```
 
+`POST /api/v1/sync` on any `siteServer` pulls `segments` / `operations` / `measurements` / `alerts` from `SYNC_SITES` / `EDGE_SITES` / `CENTRAL_PLANT_URL`. Body: `{ site, from, to, machines?, kinds? }`. Optional `SYNC_TOKEN`. Does not republish to RabbitMQ.
+
+```javascript
+import { siteSync } from '@yarkivaev/scada';
+
+await siteSync({ sites, queryFor, targets }).run({
+  site: 'edge-icht-1',
+  from,
+  to,
+  kinds: ['segments']
+});
+```
+
 Downstream plant packages should pin a released tag (e.g. `#v2.3.46`).
 
 Alert and HMI copy are inject-only: pass `translations` into `siteServer` / `plantServer` / `alertPipeline` config, and override `TAG_CATALOG_PATH` or `plantApi({ tagCatalog })` for site taxonomy. Defaults ship empty or English stubs.

package/index.js

@@ -80,3 +80,7 @@ export { default as exportQuery } from './src/application/export/exportQuery.js'
 export { default as exportStream } from './src/application/export/exportStream.js';
 export { default as exportSink } from './src/application/export/exportSink.js';
 export { default as exportJob } from './src/application/export/exportJob.js';
+export { default as siteSync } from './src/application/sync/siteSync.js';
+export { default as siteSyncSites } from './src/application/sync/siteSyncSites.js';
+export { default as siteSyncTargets } from './src/application/sync/siteSyncTargets.js';
+export { default as siteSyncBind } from './src/application/sync/siteSyncBind.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.60",
+  "version": "2.3.61",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",

package/src/application/siteServer.js

@@ -10,6 +10,7 @@ import { metricsSinkFromPool } from '../infrastructure/persistence/pg/metrics.js
 import startTelemetryIngest from './siteTelemetry.js';
 import timelineOperatorFromEnv from './timelineOperatorFromEnv.js';
 import { buildSiteOperatorCatalog } from './siteOperatorCatalog.js';
+import siteSyncBind from './sync/siteSyncBind.js';
 
 function stompFromEnv(env) {
     return {
@@ -100,10 +101,10 @@ function plantFactoryWithOperations(plantFactory, ops, sink) {
     };
 }
 
-function siteExtraRoutes(catalog, extraRoutes) {
+function siteExtraRoutes(catalog, extraRoutes, syncRoutes) {
     return (path, plant, clock) => {
         const userExtra = extraRoutes ? extraRoutes(path, plant, clock) : [];
-        return [...catalog.routes, ...userExtra];
+        return [...catalog.routes, ...syncRoutes, ...userExtra];
     };
 }
 
@@ -145,6 +146,7 @@ export default async function siteServer(config) {
     if (catalog.sync) {
         await catalog.sync.start();
     }
+    const syncRoutes = siteSyncBind(env, sink, ops, { topic: config.topic, basePath });
     const plant = await plantServer({
         port: config.port || parseInt(env.PORT || '3000', 10),
         basePath,
@@ -152,7 +154,7 @@ export default async function siteServer(config) {
         requirePool: config.requirePool,
         stomp: stompFromEnv(env),
         plantFactory: plantFactoryWithOperations(config.plantFactory, ops, sink),
-        extraRoutes: siteExtraRoutes(catalog, config.extraRoutes),
+        extraRoutes: siteExtraRoutes(catalog, config.extraRoutes, syncRoutes),
         timelineOperator: timelineOperatorFromEnv(catalog, env, config),
         operationDecisions: catalog.decisions,
         owners: config.owners,

package/src/application/sync/siteSync.js

@@ -0,0 +1,102 @@
+import { listedItems } from './siteSyncRows.js';
+import { siteById } from './siteSyncSites.js';
+
+/**
+ * Pulls selected kinds from a remote plantApi into local targets.
+ *
+ * @param {object} ports - { sites, queryFor, targets }
+ * @returns {object} frozen sync with run(request)
+ *
+ * @example
+ *   const sync = siteSync({ sites, queryFor, targets });
+ *   await sync.run({ site: 'edge-icht-1', from, to, machines: ['icht1'], kinds: ['segments'] });
+ */
+function pull(query, kind, machine, range) {
+    if (kind === 'segments') {
+        return query.segments(machine, range);
+    }
+    if (kind === 'operations') {
+        return query.operations(machine, range);
+    }
+    if (kind === 'measurements') {
+        return query.measurements(machine, range);
+    }
+    if (kind === 'alerts') {
+        return query.alerts(machine, range);
+    }
+    throw new Error(`site sync cannot pull unknown kind ${kind}`);
+}
+
+function kindsOf(request, targets) {
+    const selected = request.kinds && request.kinds.length > 0 ? request.kinds : Object.keys(targets);
+    selected.forEach((kind) => {
+        if (!targets[kind]) {
+            throw new Error(`site sync unknown kind ${kind}`);
+        }
+    });
+    return selected;
+}
+
+function machinesOf(request, site, query) {
+    if (request.machines && request.machines.length > 0) {
+        return Promise.resolve(request.machines);
+    }
+    if (site.machines && site.machines.length > 0) {
+        return Promise.resolve(site.machines);
+    }
+    return query.machines().then((body) => {
+        return listedItems(body).map((item) => {
+            return item.id;
+        });
+    });
+}
+
+function zero(kinds) {
+    return Object.fromEntries(kinds.map((kind) => {
+        return [kind, 0];
+    }));
+}
+
+function jobsOf(machines, kinds) {
+    return machines.flatMap((machine) => {
+        return kinds.map((kind) => {
+            return { machine, kind };
+        });
+    });
+}
+
+function add(counts, kind, added) {
+    return { ...counts, [kind]: counts[kind] + added };
+}
+
+function writeKind(query, targets, job, range) {
+    return pull(query, job.kind, job.machine, range).then((body) => {
+        return targets[job.kind].write(job.machine, body);
+    });
+}
+
+function fill(query, targets, kinds, machines, range) {
+    return jobsOf(machines, kinds).reduce((chain, job) => {
+        return chain.then((counts) => {
+            return writeKind(query, targets, job, range).then((added) => {
+                return add(counts, job.kind, added);
+            });
+        });
+    }, Promise.resolve(zero(kinds)));
+}
+
+export default function siteSync(ports) {
+    return Object.freeze({
+        async run(request) {
+            if (!request.from || !request.to) {
+                throw new Error('site sync requires from and to');
+            }
+            const site = siteById(ports.sites, request.site);
+            const query = ports.queryFor(site);
+            const kinds = kindsOf(request, ports.targets);
+            const machines = await machinesOf(request, site, query);
+            const counts = await fill(query, ports.targets, kinds, machines, request);
+            return { site: site.id, machines, counts };
+        }
+    });
+}

package/src/application/sync/siteSyncBind.js

@@ -0,0 +1,64 @@
+import { clickhouseSink } from '@yarkivaev/source-to-sink';
+import exportQuery from '../export/exportQuery.js';
+import scadaClient from '../../infrastructure/client/scadaClient.js';
+import { metricsSinkFromPool } from '../../infrastructure/persistence/pg/metrics.js';
+import siteSync from './siteSync.js';
+import siteSyncSites from './siteSyncSites.js';
+import siteSyncTargets from './siteSyncTargets.js';
+import syncRoute from '../../infrastructure/http/plant/routes/syncRoute.js';
+
+/**
+ * Wires siteSync routes from env, sink persistence, and an optional topic port.
+ *
+ * @param {object} env - process environment
+ * @param {object} sink - supervisor sink with pool and dataAccess
+ * @param {object} operations - operations port
+ * @param {object} [extras] - topic(machine, key), basePath
+ * @returns {object[]} plant extra routes, empty when no sites are configured
+ *
+ * @example
+ *   siteSyncBind(env, sink, ops, { topic, basePath: '/api/v1' });
+ */
+function fetchWithToken(token) {
+    if (!token) {
+        return fetch;
+    }
+    return (url, options) => {
+        const headers = { ...(options && options.headers), Authorization: `Bearer ${token}` };
+        return fetch(url, { ...options, headers });
+    };
+}
+
+function metricsOf(sink, env) {
+    if (env.SINK_DB_PROFILE !== 'central') {
+        return metricsSinkFromPool(sink.pool);
+    }
+    const url = env.CLICKHOUSE_URL
+        || (env.CLICKHOUSE_HOST ? `http://${env.CLICKHOUSE_HOST}:8123` : undefined);
+    if (!url) {
+        return undefined;
+    }
+    return clickhouseSink(url, 'scada.metrics');
+}
+
+function queryFor(site) {
+    return exportQuery(scadaClient(site.url, fetchWithToken(site.token)));
+}
+
+export default function siteSyncBind(env, sink, operations, extras) {
+    const sites = siteSyncSites(env);
+    if (sites.length === 0) {
+        return [];
+    }
+    const extra = extras || {};
+    const targets = siteSyncTargets({
+        postgres: env.POSTGRES_URL,
+        pool: sink.pool,
+        operations,
+        alerts: sink.dataAccess.alerts,
+        metrics: metricsOf(sink, env),
+        topic: extra.topic
+    });
+    const sync = siteSync({ sites, queryFor, targets });
+    return syncRoute(extra.basePath || '/api/v1', sync, env.SYNC_TOKEN);
+}

package/src/application/sync/siteSyncRows.js

@@ -0,0 +1,91 @@
+/**
+ * Maps plant API JSON into local persistence rows for site sync.
+ *
+ * @example
+ *   segmentRow('icht1', { name: 'on', start: '2026-01-01T00:00:00.000Z', end: '...', duration: 60 });
+ */
+
+/**
+ * Maps one segment API item onto the segments table shape.
+ *
+ * @param {string} machine - machine id
+ * @param {object} item - plant API segment
+ * @returns {object} persistence row
+ */
+export function segmentRow(machine, item) {
+    return {
+        machine,
+        kind: item.kind || 'phase',
+        name: item.name,
+        start_time: item.start || item.start_time,
+        end_time: item.end || item.end_time,
+        duration: item.duration,
+        options: item.options,
+        tags: item.tags,
+        properties: item.properties,
+        resolved: item.resolved !== false
+    };
+}
+
+/**
+ * Maps one operation API item onto the operations port shape.
+ *
+ * @param {string} machine - machine id fallback
+ * @param {object} item - plant API operation
+ * @returns {object} persistence item
+ */
+export function operationRow(machine, item) {
+    return {
+        machine: item.machine || machine,
+        key: item.external_key || item.key,
+        occurred_at: item.occurred_at,
+        kind: item.kind,
+        payload: item.payload || {}
+    };
+}
+
+/**
+ * Maps measurement series onto metrics sink records via a topic port.
+ *
+ * @param {string} machine - machine id
+ * @param {object} body - plant API { items: [{ key, values }] }
+ * @param {function} topic - (machine, key) => MQTT/CH topic
+ * @returns {object[]} records with topic, ts, value
+ */
+export function measurementRows(machine, body, topic) {
+    const items = body.items || [];
+    return items.flatMap((series) => {
+        const dest = topic(machine, series.key);
+        return (series.values || []).map((point) => {
+            return { topic: dest, ts: point.timestamp, value: point.value };
+        });
+    });
+}
+
+/**
+ * Maps one alert API item onto the alerts table shape.
+ *
+ * @param {string} machine - machine id
+ * @param {object} item - plant API alert
+ * @returns {object} persistence row
+ */
+export function alertRow(machine, item) {
+    return {
+        name: item.name,
+        message: item.message,
+        machine,
+        severity: item.severity || 'warning',
+        timestamp: item.timestamp,
+        acknowledged: item.acknowledged === true
+    };
+}
+
+export function listedItems(body) {
+    if (Array.isArray(body)) {
+        return body;
+    }
+    if (body && Array.isArray(body.items)) {
+        return body.items;
+    }
+    return [];
+}

package/src/application/sync/siteSyncSites.js

@@ -0,0 +1,81 @@
+/**
+ * Builds the allowlist of remote plant APIs a site may pull from.
+ *
+ * Reads SYNC_SITES, EDGE_SITES, and CENTRAL_PLANT_URL. First id wins.
+ *
+ * @param {NodeJS.ProcessEnv} env - process environment
+ * @returns {object[]} frozen sites with id, url, optional token and machines
+ *
+ * @example
+ *   siteSyncSites({ SYNC_SITES: '[{"id":"edge-icht-1","url":"http://edge/api/v1"}]' });
+ */
+function parseJson(raw, label) {
+    try {
+        return JSON.parse(raw);
+    } catch (cause) {
+        throw new Error(`${label} must be valid JSON: ${cause.message}`);
+    }
+}
+
+function asSite(raw, index, label) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+        throw new Error(`${label}[${index}] must be an object`);
+    }
+    const url = raw.url || raw.baseUrl;
+    if (typeof url !== 'string' || url.length === 0) {
+        throw new Error(`${label}[${index}] requires url`);
+    }
+    const id = raw.id || (Array.isArray(raw.machines) && raw.machines[0]) || `${label}-${index}`;
+    const site = { id, url };
+    if (typeof raw.token === 'string' && raw.token.length > 0) {
+        site.token = raw.token;
+    }
+    if (Array.isArray(raw.machines) && raw.machines.length > 0) {
+        site.machines = raw.machines;
+    }
+    return Object.freeze(site);
+}
+
+function parseList(raw, label) {
+    const parsed = parseJson(raw, label);
+    if (!Array.isArray(parsed)) {
+        throw new Error(`${label} must be a JSON array`);
+    }
+    return parsed.map((item, index) => {
+        return asSite(item, index, label);
+    });
+}
+
+function merge(lists) {
+    const seen = new Map();
+    lists.flat().forEach((site) => {
+        if (!seen.has(site.id)) {
+            seen.set(site.id, site);
+        }
+    });
+    return [...seen.values()];
+}
+
+export default function siteSyncSites(env) {
+    const lists = [];
+    if (typeof env.SYNC_SITES === 'string' && env.SYNC_SITES.length > 0) {
+        lists.push(parseList(env.SYNC_SITES, 'SYNC_SITES'));
+    }
+    if (typeof env.EDGE_SITES === 'string' && env.EDGE_SITES.length > 0) {
+        lists.push(parseList(env.EDGE_SITES, 'EDGE_SITES'));
+    }
+    if (typeof env.CENTRAL_PLANT_URL === 'string' && env.CENTRAL_PLANT_URL.length > 0) {
+        lists.push([Object.freeze({ id: 'central', url: env.CENTRAL_PLANT_URL })]);
+    }
+    return merge(lists);
+}
+
+export function siteById(sites, id) {
+    const found = sites.find((site) => {
+        return site.id === id;
+    });
+    if (!found) {
+        throw new Error(`site sync unknown site ${id}`);
+    }
+    return found;
+}

package/src/application/sync/siteSyncTargets.js

@@ -0,0 +1,98 @@
+import { postgresSink } from '@yarkivaev/source-to-sink';
+import {
+    alertRow,
+    listedItems,
+    measurementRows,
+    operationRow,
+    segmentRow
+} from './siteSyncRows.js';
+import {
+    segmentColumns,
+    segmentConflict,
+    segmentUpdateColumns
+} from '../../infrastructure/ingest/pipelines/segmentPipeline.js';
+
+/**
+ * Default local write targets for site sync kinds.
+ *
+ * @param {object} ports - pool, postgres, operations, alerts, metrics, topic
+ * @returns {object} kind → { write(machine, body) }
+ *
+ * @example
+ *   siteSyncTargets({ operations, postgres, pool, alerts, metrics, topic });
+ */
+function segmentTarget(postgres, pool) {
+    const sink = postgresSink(postgres, 'segments', segmentColumns, {
+        pool,
+        conflict: segmentConflict,
+        update: segmentUpdateColumns
+    });
+    return {
+        async write(machine, body) {
+            const rows = listedItems(body).map((item) => {
+                return segmentRow(machine, item);
+            });
+            if (rows.length > 0) {
+                await sink.write(rows);
+            }
+            return rows.length;
+        }
+    };
+}
+
+function operationTarget(operations) {
+    return {
+        async write(machine, body) {
+            const rows = listedItems(body).map((item) => {
+                return operationRow(machine, item);
+            });
+            if (rows.length > 0) {
+                await operations.upsertMany(rows);
+            }
+            return rows.length;
+        }
+    };
+}
+
+function measurementTarget(metrics, topic) {
+    return {
+        async write(machine, body) {
+            const rows = measurementRows(machine, body, topic);
+            if (rows.length > 0) {
+                await metrics.write(rows);
+            }
+            return rows.length;
+        }
+    };
+}
+
+function alertTarget(alerts) {
+    return {
+        async write(machine, body) {
+            const rows = listedItems(body).map((item) => {
+                return alertRow(machine, item);
+            });
+            await Promise.all(rows.map((row) => {
+                return alerts.put(row);
+            }));
+            return rows.length;
+        }
+    };
+}
+
+export default function siteSyncTargets(ports) {
+    const targets = {};
+    if (ports.postgres && ports.pool) {
+        targets.segments = segmentTarget(ports.postgres, ports.pool);
+    }
+    if (ports.operations) {
+        targets.operations = operationTarget(ports.operations);
+    }
+    if (ports.metrics && ports.topic) {
+        targets.measurements = measurementTarget(ports.metrics, ports.topic);
+    }
+    if (ports.alerts && typeof ports.alerts.put === 'function') {
+        targets.alerts = alertTarget(ports.alerts);
+    }
+    return Object.freeze(targets);
+}

package/src/infrastructure/http/plant/routes/syncRoute.js

@@ -0,0 +1,50 @@
+import { errorResponse, jsonResponse, readBody, route } from '@yarkivaev/simple-server';
+
+/**
+ * POST /sync — pull selected kinds from an allowlisted remote plantApi.
+ *
+ * @param {string} basePath - API prefix
+ * @param {object} sync - siteSync port with run(request)
+ * @param {string} [token] - optional Bearer token
+ * @returns {object[]} routes
+ *
+ * @example
+ *   syncRoute('/api/v1', sync, process.env.SYNC_TOKEN);
+ */
+function rejectAuth(token, req, res) {
+    if (!token) {
+        return false;
+    }
+    if (req.headers.authorization === `Bearer ${token}`) {
+        return false;
+    }
+    errorResponse('UNAUTHORIZED', 'site sync token is required', 401).send(res);
+    return true;
+}
+
+function rejectScope(parsed, res) {
+    if (parsed.site && parsed.from && parsed.to) {
+        return false;
+    }
+    errorResponse('BAD_REQUEST', 'site sync requires site from and to', 400).send(res);
+    return true;
+}
+
+export default function syncRoute(basePath, sync, token) {
+    return [
+        route('POST', `${basePath}/sync`, async (req, res) => {
+            if (rejectAuth(token, req, res)) {
+                return;
+            }
+            const parsed = JSON.parse(await readBody(req));
+            if (rejectScope(parsed, res)) {
+                return;
+            }
+            try {
+                jsonResponse(await sync.run(parsed)).send(res);
+            } catch (error) {
+                errorResponse('BAD_REQUEST', error.message, 400).send(res);
+            }
+        })
+    ];
+}

package/src/infrastructure/persistence/memory/alerts.js

@@ -2,10 +2,21 @@
  * In-memory alert hydration port for tests and local runs.
  *
  * @param {object} store - shared mutable store with alerts array
- * @returns {object} alerts port with listUnacknowledged
+ * @returns {object} alerts port with put and listUnacknowledged
  */
 export default function alertsStateMemory(store) {
     return {
+        put(row) {
+            const found = store.alerts.find((item) => {
+                return item.machine === row.machine && item.timestamp === row.timestamp && item.name === row.name;
+            });
+            if (found) {
+                return found;
+            }
+            const created = { id: store.alerts.length + 1, ...row };
+            store.alerts.push(created);
+            return created;
+        },
         listUnacknowledged(filters) {
             let rows = store.alerts.filter((row) => {
                 return row.acknowledged === false;

package/src/infrastructure/persistence/pg/alerts.js

@@ -2,7 +2,7 @@
  * PostgreSQL alert hydration port for supervisor-sink and plant STOMP alerts.
  *
  * @param {object} pool - pg pool
- * @returns {object} alerts port with listUnacknowledged
+ * @returns {object} alerts port with put and listUnacknowledged
  *
  * @example
  *   const store = alertsStatePg(pool);
@@ -10,6 +10,21 @@
  */
 export default function alertsStatePg(pool) {
     return {
+        async put(row) {
+            const existing = await pool.query(
+                'SELECT id FROM alerts WHERE machine = $1 AND timestamp = $2 AND name = $3',
+                [row.machine, row.timestamp, row.name]
+            );
+            if (existing.rows.length > 0) {
+                return existing.rows[0];
+            }
+            const inserted = await pool.query(
+                `INSERT INTO alerts (name, message, machine, severity, timestamp, acknowledged)
+                 VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`,
+                [row.name, row.message, row.machine, row.severity, row.timestamp, row.acknowledged === true]
+            );
+            return inserted.rows[0];
+        },
         async listUnacknowledged(filters) {
             let sql = 'SELECT * FROM alerts WHERE acknowledged = FALSE';
             const prm = [];