@yarkivaev/scada 2.3.116 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -0
- package/charts/scada/templates/processes.yaml +22 -0
- package/charts/scada/values.schema.json +12 -0
- package/db/migrations/V0009__collector_task_state.sql +6 -0
- package/package.json +11 -5
- package/src/application/deploy/readDocument.js +4 -1
- package/src/application/plantServer.js +1 -1
- package/src/application/runtime/additionalAdapters.js +139 -0
- package/src/application/runtime/assignment.js +30 -0
- package/src/application/runtime/budgetedSink.js +17 -0
- package/src/application/runtime/collectorSignals.js +14 -0
- package/src/application/runtime/compositeLifecycle.js +49 -0
- package/src/application/runtime/createRegistry.js +20 -0
- package/src/application/runtime/deadline.js +8 -0
- package/src/application/runtime/index.js +10 -0
- package/src/application/runtime/manifest.js +38 -0
- package/src/application/runtime/pipelineAdapters.js +60 -0
- package/src/application/runtime/resolveSecrets.js +12 -0
- package/src/application/runtime/resourceClaims.js +21 -0
- package/src/application/runtime/runCollector.js +50 -0
- package/src/application/runtime/runtime.js +86 -0
- package/src/application/runtime/runtimeHealth.js +29 -0
- package/src/application/runtime/runtimeServices.js +15 -0
- package/src/application/runtime/stateDocument.js +22 -0
- package/src/application/runtime/streamPorts.js +42 -0
- package/src/application/runtime/streamTask.js +105 -0
- package/src/application/runtime/taskBudget.js +31 -0
- package/src/application/runtime/taskContext.js +18 -0
- package/src/application/runtime/taskQueue.js +21 -0
- package/src/application/runtime/taskSlot.js +46 -0
- package/src/application/runtime/taskState.js +73 -0
- package/src/application/siteCollection.js +14 -0
- package/src/application/siteServer.js +7 -6
- package/src/application/siteTelemetry.js +1 -1
- package/src/application/supervisorSink.js +9 -8
- package/src/application/sync/siteSyncBind.js +1 -1
- package/src/application/sync/siteSyncTargets.js +1 -1
- package/src/bin/collector.js +7 -0
- package/src/bin/modbus-to-mqtt.js +11 -9
- package/src/infrastructure/ingest/activity/activityTracking.js +15 -21
- package/src/infrastructure/ingest/activity/activityTransformer.js +16 -15
- package/src/infrastructure/ingest/durableConsume.js +2 -2
- package/src/infrastructure/ingest/ingestBatch.js +13 -0
- package/src/infrastructure/ingest/modbus/mx210Tcp.js +4 -4
- package/src/infrastructure/ingest/modbus/silentStreams.js +33 -8
- package/src/infrastructure/ingest/mqtt/metricsTransformer.js +4 -5
- package/src/infrastructure/ingest/mqtt/modbusMqtt.js +17 -19
- package/src/infrastructure/ingest/mqtt/modbusPublish.js +4 -0
- package/src/infrastructure/ingest/mqtt/mqttMetrics.js +17 -21
- package/src/infrastructure/ingest/mqtt/registerPublish.js +3 -3
- package/src/infrastructure/ingest/pipelines/alertPipeline.js +7 -6
- package/src/infrastructure/ingest/pipelines/decisionPipeline.js +7 -6
- package/src/infrastructure/ingest/pipelines/segmentPipeline.js +8 -17
- package/src/infrastructure/ingest/pipelines/stompOptions.js +7 -0
- package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js +10 -11
- package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js +10 -6
- package/src/infrastructure/ingest/telemetry/foldingSink.js +5 -2
- package/src/infrastructure/messaging/stomp/segmentRetags.js +1 -1
- package/src/infrastructure/messaging/stomp/userDecisions.js +1 -1
- package/src/infrastructure/opcua/certificates.js +11 -0
- package/src/infrastructure/opcua/delivery.js +38 -0
- package/src/infrastructure/opcua/endpoint.js +12 -2
- package/src/infrastructure/opcua/session.js +20 -10
- package/src/infrastructure/opcua/source.js +6 -3
- package/src/infrastructure/persistence/runtime/taskStatePg.js +24 -0
- package/src/infrastructure/streams/README.md +291 -0
- package/src/infrastructure/streams/abortable.js +14 -0
- package/src/infrastructure/streams/amqpSink/publication.js +46 -0
- package/src/infrastructure/streams/amqpSink/session.js +105 -0
- package/src/infrastructure/streams/amqpSink.js +47 -0
- package/src/infrastructure/streams/amqpSource/session.js +127 -0
- package/src/infrastructure/streams/amqpSource.js +97 -0
- package/src/infrastructure/streams/brokerClose.js +19 -0
- package/src/infrastructure/streams/budgetAccept.js +6 -0
- package/src/infrastructure/streams/clickhouseSink.js +48 -0
- package/src/infrastructure/streams/deadline.js +1 -0
- package/src/infrastructure/streams/follow.js +41 -0
- package/src/infrastructure/streams/index.js +15 -0
- package/src/infrastructure/streams/lokiSource.js +88 -0
- package/src/infrastructure/streams/modbusConnection.js +173 -0
- package/src/infrastructure/streams/modbusRtuBusSource.js +124 -0
- package/src/infrastructure/streams/modbusSource.js +185 -0
- package/src/infrastructure/streams/mqttSink/session.js +51 -0
- package/src/infrastructure/streams/mqttSink.js +54 -0
- package/src/infrastructure/streams/mqttSource.js +66 -0
- package/src/infrastructure/streams/pollingSource.js +1 -0
- package/src/infrastructure/streams/positive.js +5 -0
- package/src/infrastructure/streams/postgresSink.js +124 -0
- package/src/infrastructure/streams/serialReceive.js +80 -0
- package/src/infrastructure/streams/serialSource.js +87 -0
- package/src/infrastructure/streams/sqliteSink.js +55 -0
- package/src/infrastructure/streams/stompBody.js +46 -0
- package/src/infrastructure/streams/stompSend.js +89 -0
- package/src/infrastructure/streams/stompSource.js +201 -0
- package/src/infrastructure/sync/operationSyncIngest.js +1 -1
|
@@ -15,7 +15,7 @@ const appRoot = path.dirname(fileURLToPath(import.meta.url));
|
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Reads deployment environment into a normalized config object.
|
|
18
|
-
* `SINK_PIPELINES=false` leaves Postgres and the state HTTP API in this process
|
|
18
|
+
* `COLLECTOR_MODE=external` or `SINK_PIPELINES=false` leaves Postgres and the state HTTP API in this process
|
|
19
19
|
* and leaves the STOMP consumers to another container.
|
|
20
20
|
*
|
|
21
21
|
* @param {NodeJS.ProcessEnv} env - process environment
|
|
@@ -41,7 +41,7 @@ export function readDeploymentConfig(env) {
|
|
|
41
41
|
retentionDays: parseInt(env.RETENTION_DAYS || '30', 10),
|
|
42
42
|
migrationsDir: path.join(root, 'db', 'migrations'),
|
|
43
43
|
requestTimeoutMs: parseRequestTimeoutMs(env.REQUEST_TIMEOUT_MS),
|
|
44
|
-
pipelines: env.SINK_PIPELINES !== 'false',
|
|
44
|
+
pipelines: env.COLLECTOR_MODE !== 'external' && env.SINK_PIPELINES !== 'false',
|
|
45
45
|
pipeline: {
|
|
46
46
|
prefetch,
|
|
47
47
|
deadLetter: env.SEGMENTS_DEAD_LETTER_ENABLED === 'true',
|
|
@@ -60,6 +60,12 @@ export function readDeploymentConfig(env) {
|
|
|
60
60
|
};
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
function startPipelines(cfg, pool) {
|
|
64
|
+
const pipelines = [segmentPipeline(cfg.stomp, cfg.postgres, pool, cfg.pipeline),
|
|
65
|
+
decisionPipeline(cfg.stomp, pool, cfg.pipeline), alertPipeline(cfg.stomp, pool, cfg.pipeline)];
|
|
66
|
+
return Promise.all(pipelines.map((pipeline) => { return pipeline.start(); }));
|
|
67
|
+
}
|
|
68
|
+
|
|
63
69
|
/**
|
|
64
70
|
* Wires STOMP pipelines, migrations, and state data access from env config.
|
|
65
71
|
* HTTP binding uses edgeApi (metrics, checkpoint, retention).
|
|
@@ -70,9 +76,6 @@ export function readDeploymentConfig(env) {
|
|
|
70
76
|
export default function supervisorSink(env = process.env) {
|
|
71
77
|
const cfg = readDeploymentConfig(env);
|
|
72
78
|
const pool = holdClient(new pg.Pool({ connectionString: cfg.postgres }));
|
|
73
|
-
const segments = segmentPipeline(cfg.stomp, cfg.postgres, pool, cfg.pipeline);
|
|
74
|
-
const decisions = decisionPipeline(cfg.stomp, pool, cfg.pipeline);
|
|
75
|
-
const alerts = alertPipeline(cfg.stomp, pool, cfg.pipeline);
|
|
76
79
|
const metricsEnabled = cfg.sinkDbProfile === 'edge';
|
|
77
80
|
const dataAccess = stateDataFromPool(pool, { metricsEnabled });
|
|
78
81
|
return {
|
|
@@ -97,9 +100,7 @@ export default function supervisorSink(env = process.env) {
|
|
|
97
100
|
await adminPool.end();
|
|
98
101
|
}
|
|
99
102
|
if (cfg.pipelines) {
|
|
100
|
-
await
|
|
101
|
-
decisions.start();
|
|
102
|
-
alerts.start();
|
|
103
|
+
await startPipelines(cfg, pool);
|
|
103
104
|
}
|
|
104
105
|
if (httpServer) {
|
|
105
106
|
httpServer.start();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { clickhouseSink } from '
|
|
1
|
+
import { clickhouseSink } from '../../infrastructure/streams/index.js';
|
|
2
2
|
import { jobRoutes, jobs } from '@yarkivaev/simple-server';
|
|
3
3
|
import exportQuery from '../export/exportQuery.js';
|
|
4
4
|
import scadaClient from '../../infrastructure/client/scadaClient.js';
|
|
@@ -3,12 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Polls ADDRESS or MODBUS_DEVICES and publishes STREAM + MAPPING document paths.
|
|
5
5
|
*/
|
|
6
|
-
|
|
7
|
-
process.stderr.write(`Unhandled rejection: ${String(err)}\n`);
|
|
8
|
-
process.exit(1);
|
|
9
|
-
});
|
|
10
|
-
|
|
11
|
-
import modbusPublish from '../infrastructure/ingest/mqtt/modbusPublish.js';
|
|
6
|
+
import { createRegistry, runCollector } from '../application/runtime/index.js';
|
|
12
7
|
|
|
13
8
|
/**
|
|
14
9
|
* Parses a JSON array environment value.
|
|
@@ -41,7 +36,7 @@ function integer(text) {
|
|
|
41
36
|
}
|
|
42
37
|
|
|
43
38
|
const addresses = jsonList(process.env.ADDRESS);
|
|
44
|
-
const
|
|
39
|
+
const config = {
|
|
45
40
|
address: integer(process.env.MODBUS_ADDRESS),
|
|
46
41
|
addresses,
|
|
47
42
|
clientId: process.env.CLIENT_ID,
|
|
@@ -52,6 +47,13 @@ const pipeline = modbusPublish(process.env.MQTT_URL, {
|
|
|
52
47
|
streams: jsonList(process.env.STREAM) || [],
|
|
53
48
|
threshold: parseInt(process.env.CIRCUIT_THRESHOLD || '5', 10),
|
|
54
49
|
timeout: parseInt(process.env.CIRCUIT_TIMEOUT || '60', 10)
|
|
55
|
-
}
|
|
50
|
+
};
|
|
56
51
|
|
|
57
|
-
|
|
52
|
+
try {
|
|
53
|
+
const env = { ...process.env, COLLECTOR_MANIFEST: JSON.stringify({ assignments: [{ taskId: 'modbus', generation: 0, specVersion: 1,
|
|
54
|
+
spec: { adapter: 'modbus', url: { $secret: 'MQTT_URL' }, config } }] }) };
|
|
55
|
+
await runCollector({ registry: createRegistry(), env, argv: [] });
|
|
56
|
+
} catch (error) {
|
|
57
|
+
process.stderr.write(`collector: ${error.name}\n`);
|
|
58
|
+
process.exitCode = 1;
|
|
59
|
+
}
|
|
@@ -1,19 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
clickhouseSink,
|
|
5
|
-
clock,
|
|
6
|
-
lokiSource,
|
|
7
|
-
timedBatch
|
|
8
|
-
} from '@yarkivaev/source-to-sink';
|
|
1
|
+
import { clock } from '@yarkivaev/source-to-sink';
|
|
2
|
+
import { clickhouseSink, lokiSource } from '../../streams/index.js';
|
|
3
|
+
import ingestBatch from '../ingestBatch.js';
|
|
9
4
|
import activityCodec from './activityTransformer.js';
|
|
10
5
|
|
|
11
6
|
/**
|
|
12
7
|
* Pipeline for streaming Loki HTTP access logs to ClickHouse.
|
|
13
8
|
*
|
|
14
|
-
* Polls Loki
|
|
15
|
-
* before
|
|
16
|
-
*
|
|
9
|
+
* Polls Loki and durably inserts each accepted log into ClickHouse
|
|
10
|
+
* before advancing the serial source cursor.
|
|
11
|
+
* Bounds pending records and bytes and waits for durable writes.
|
|
17
12
|
*
|
|
18
13
|
* @example
|
|
19
14
|
* const pipeline = activityTracking(
|
|
@@ -31,10 +26,10 @@ import activityCodec from './activityTransformer.js';
|
|
|
31
26
|
* @param {string} query - LogQL query string
|
|
32
27
|
* @param {object} config - Pipeline configuration
|
|
33
28
|
* @param {number} config.poll - Loki polling interval in seconds
|
|
34
|
-
* @param {number} config.size -
|
|
29
|
+
* @param {number} config.size - Legacy option; serial durable acceptance writes one record at a time
|
|
35
30
|
* @param {number} config.interval - Seconds before time-based flush
|
|
36
|
-
* @param {number} config.threshold -
|
|
37
|
-
* @param {number} config.timeout -
|
|
31
|
+
* @param {number} config.threshold - Legacy option (retained for configuration compatibility)
|
|
32
|
+
* @param {number} config.timeout - Legacy option (use writeTimeoutMs for write deadlines)
|
|
38
33
|
* @returns {object} Pipeline with start() and stop() methods
|
|
39
34
|
*/
|
|
40
35
|
export default function activityTracking(loki, clickhouse, query, config) {
|
|
@@ -51,24 +46,23 @@ export default function activityTracking(loki, clickhouse, query, config) {
|
|
|
51
46
|
throw new Error('Config must be an object');
|
|
52
47
|
}
|
|
53
48
|
const clk = clock();
|
|
54
|
-
const breaker = circuit(config.threshold, config.timeout, clk);
|
|
55
49
|
const sink = clickhouseSink(clickhouse, 'activity.http_access');
|
|
56
|
-
const collector =
|
|
50
|
+
const collector = ingestBatch(sink, { ...config, size: 1 });
|
|
57
51
|
const transformer = activityCodec(collector);
|
|
58
|
-
const source = lokiSource(loki, query, config.poll, transformer, clk);
|
|
52
|
+
const source = lokiSource(loki, query, config.poll, transformer, clk, { ...config.lokiOptions, budget: config.budget });
|
|
59
53
|
return {
|
|
60
54
|
/**
|
|
61
55
|
* Starts the pipeline.
|
|
62
56
|
*/
|
|
63
57
|
start() {
|
|
64
|
-
source.start();
|
|
58
|
+
return source.start();
|
|
65
59
|
},
|
|
66
60
|
/**
|
|
67
61
|
* Stops the pipeline.
|
|
68
62
|
*/
|
|
69
|
-
stop() {
|
|
70
|
-
source.stop();
|
|
71
|
-
collector.stop();
|
|
63
|
+
async stop() {
|
|
64
|
+
await source.stop();
|
|
65
|
+
try { await collector.stop(); } finally { await sink.stop?.(); }
|
|
72
66
|
}
|
|
73
67
|
};
|
|
74
68
|
}
|
|
@@ -22,24 +22,25 @@ export default function activityCodec(collector) {
|
|
|
22
22
|
* @param {object} entry - Log entry with ts and line
|
|
23
23
|
*/
|
|
24
24
|
accept(entry) {
|
|
25
|
+
let log;
|
|
25
26
|
try {
|
|
26
|
-
|
|
27
|
-
if (!log.StartUTC || !log.RequestMethod || !log.RequestPath) {
|
|
28
|
-
return;
|
|
29
|
-
}
|
|
30
|
-
collector.accept({
|
|
31
|
-
ts: new Date(log.StartUTC).getTime(),
|
|
32
|
-
user: log['request_Remote-User'] || '',
|
|
33
|
-
email: log['request_Remote-Email'] || '',
|
|
34
|
-
method: log.RequestMethod || '',
|
|
35
|
-
path: log.RequestPath || '',
|
|
36
|
-
status: parseInt(log.DownstreamStatus, 10) || 0,
|
|
37
|
-
duration: parseFloat(log.Duration) / 1e9 || 0,
|
|
38
|
-
ip: log.ClientHost || ''
|
|
39
|
-
});
|
|
27
|
+
log = JSON.parse(entry.line);
|
|
40
28
|
} catch {
|
|
41
|
-
|
|
29
|
+
return undefined;
|
|
42
30
|
}
|
|
31
|
+
if (!log || !log.StartUTC || !log.RequestMethod || !log.RequestPath) {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
return collector.accept({
|
|
35
|
+
ts: new Date(log.StartUTC).getTime(),
|
|
36
|
+
user: log['request_Remote-User'] || '',
|
|
37
|
+
email: log['request_Remote-Email'] || '',
|
|
38
|
+
method: log.RequestMethod || '',
|
|
39
|
+
path: log.RequestPath || '',
|
|
40
|
+
status: parseInt(log.DownstreamStatus, 10) || 0,
|
|
41
|
+
duration: parseFloat(log.Duration) / 1e9 || 0,
|
|
42
|
+
ip: log.ClientHost || ''
|
|
43
|
+
});
|
|
43
44
|
}
|
|
44
45
|
};
|
|
45
46
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
// Compatibility facade; transport lifecycle
|
|
2
|
-
export { amqpSession as default } from '
|
|
1
|
+
// Compatibility facade; transport lifecycle is owned by the scada stream adapters.
|
|
2
|
+
export { amqpSession as default } from '../streams/index.js';
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { boundedBatch } from '@yarkivaev/source-to-sink';
|
|
2
|
+
|
|
3
|
+
/** Shared production limits; acceptance resolves only after the sink write. */
|
|
4
|
+
export default function ingestBatch(sink, config = {}) {
|
|
5
|
+
return boundedBatch(sink, {
|
|
6
|
+
...(config.budget ? { budget: config.budget } : {}),
|
|
7
|
+
size: config.size ?? 100,
|
|
8
|
+
maxRecords: config.maxRecords ?? 1000,
|
|
9
|
+
maxBytes: config.maxBytes ?? 1048576,
|
|
10
|
+
maxWaitMs: config.maxWaitMs ?? (config.interval ?? 5) * 1000,
|
|
11
|
+
writeTimeoutMs: config.writeTimeoutMs ?? 10000
|
|
12
|
+
});
|
|
13
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { modbusSource } from '
|
|
1
|
+
import { modbusSource } from '../../streams/index.js';
|
|
2
2
|
|
|
3
3
|
const layout = [
|
|
4
4
|
{ offset: 0, channel: 'AI1' },
|
|
@@ -34,13 +34,13 @@ export function mx210Metrics(device, collector) {
|
|
|
34
34
|
return {
|
|
35
35
|
accept(registers) {
|
|
36
36
|
const ts = Date.now();
|
|
37
|
-
|
|
38
|
-
collector.accept({
|
|
37
|
+
return Promise.all(layout.map((entry) => {
|
|
38
|
+
return collector.accept({
|
|
39
39
|
topic: `MX210/${device}/GET/${entry.channel}/VALUE`,
|
|
40
40
|
ts,
|
|
41
41
|
value: decode(registers[entry.offset], registers[entry.offset + 1])
|
|
42
42
|
});
|
|
43
|
-
}
|
|
43
|
+
}));
|
|
44
44
|
}
|
|
45
45
|
};
|
|
46
46
|
}
|
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
import { clock, intervalSource } from '@yarkivaev/source-to-sink';
|
|
2
2
|
|
|
3
|
+
function track(state, pending) {
|
|
4
|
+
if (!pending || typeof pending.then !== 'function') { return; }
|
|
5
|
+
const task = pending.catch((error) => {
|
|
6
|
+
state.failure = error;
|
|
7
|
+
try { state.onError?.(error); } catch {
|
|
8
|
+
// Reporting must not turn an observed source failure into an unhandled rejection.
|
|
9
|
+
}
|
|
10
|
+
})
|
|
11
|
+
.finally(() => { state.pending.delete(task); });
|
|
12
|
+
state.pending.add(task);
|
|
13
|
+
}
|
|
14
|
+
async function drained(state) {
|
|
15
|
+
await Promise.all(state.pending);
|
|
16
|
+
if (state.failure) { throw state.failure; }
|
|
17
|
+
}
|
|
3
18
|
/**
|
|
4
19
|
* Whether a stream is past the silence budget.
|
|
5
20
|
*
|
|
@@ -27,8 +42,12 @@ function stopOne(state, name) {
|
|
|
27
42
|
if (!state.live.has(name)) {
|
|
28
43
|
return;
|
|
29
44
|
}
|
|
30
|
-
state.
|
|
31
|
-
state.
|
|
45
|
+
if (state.closing.has(name)) { return; }
|
|
46
|
+
const pending = state.held.get(name).stop();
|
|
47
|
+
if (pending && typeof pending.then === 'function') {
|
|
48
|
+
state.closing.add(name);
|
|
49
|
+
track(state, pending.finally(() => { state.live.delete(name); state.closing.delete(name); }));
|
|
50
|
+
} else { state.live.delete(name); }
|
|
32
51
|
}
|
|
33
52
|
|
|
34
53
|
/**
|
|
@@ -47,7 +66,7 @@ function startOne(state, source) {
|
|
|
47
66
|
poll = source.open(state.collector, state.clk, state.intervalSec);
|
|
48
67
|
state.held.set(name, poll);
|
|
49
68
|
}
|
|
50
|
-
poll.start();
|
|
69
|
+
track(state, poll.start());
|
|
51
70
|
state.live.add(name);
|
|
52
71
|
}
|
|
53
72
|
|
|
@@ -88,6 +107,10 @@ function gateState(config) {
|
|
|
88
107
|
lastSeen: new Map(),
|
|
89
108
|
held: new Map(),
|
|
90
109
|
live: new Set(),
|
|
110
|
+
closing: new Set(),
|
|
111
|
+
pending: new Set(),
|
|
112
|
+
failure: null,
|
|
113
|
+
onError: config.onError,
|
|
91
114
|
collector: undefined,
|
|
92
115
|
running: false
|
|
93
116
|
};
|
|
@@ -129,22 +152,24 @@ export default function silentStreams(config) {
|
|
|
129
152
|
bind(sink) {
|
|
130
153
|
state.collector = {
|
|
131
154
|
accept(record) {
|
|
132
|
-
sink.write([record]);
|
|
155
|
+
return sink.write([record]);
|
|
133
156
|
}
|
|
134
157
|
};
|
|
135
158
|
},
|
|
136
159
|
start() {
|
|
137
|
-
if (state.running) { return; }
|
|
160
|
+
if (state.running) { return drained(state); }
|
|
138
161
|
state.running = true;
|
|
139
162
|
runPulse();
|
|
140
|
-
|
|
163
|
+
track(state, ticker.start());
|
|
164
|
+
return drained(state);
|
|
141
165
|
},
|
|
142
|
-
stop() {
|
|
166
|
+
async stop() {
|
|
143
167
|
state.running = false;
|
|
144
|
-
|
|
168
|
+
track(state, ticker.stop());
|
|
145
169
|
for (const name of [...state.live]) {
|
|
146
170
|
stopOne(state, name);
|
|
147
171
|
}
|
|
172
|
+
await drained(state);
|
|
148
173
|
state.held.clear();
|
|
149
174
|
},
|
|
150
175
|
pulse: runPulse
|
|
@@ -30,25 +30,24 @@ export default function metricsCodec(collector, log = console) {
|
|
|
30
30
|
parsed = JSON.parse(raw.payload);
|
|
31
31
|
} catch {
|
|
32
32
|
log.warn(`Cannot resolve payload on ${raw.topic}: not valid JSON`);
|
|
33
|
-
return;
|
|
33
|
+
return undefined;
|
|
34
34
|
}
|
|
35
35
|
if (typeof parsed === 'number') {
|
|
36
|
-
collector.accept({
|
|
36
|
+
return collector.accept({
|
|
37
37
|
topic: raw.topic,
|
|
38
38
|
ts: Date.now(),
|
|
39
39
|
value: parsed
|
|
40
40
|
});
|
|
41
|
-
return;
|
|
42
41
|
}
|
|
43
42
|
if (typeof parsed === 'object' && parsed !== null && typeof parsed.value === 'number') {
|
|
44
|
-
collector.accept({
|
|
43
|
+
return collector.accept({
|
|
45
44
|
topic: raw.topic,
|
|
46
45
|
ts: parsed.ts || Date.now(),
|
|
47
46
|
value: parsed.value
|
|
48
47
|
});
|
|
49
|
-
return;
|
|
50
48
|
}
|
|
51
49
|
log.warn(`Cannot resolve payload on ${raw.topic}: no numeric value found`);
|
|
50
|
+
return undefined;
|
|
52
51
|
}
|
|
53
52
|
};
|
|
54
53
|
}
|
|
@@ -1,12 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
clock,
|
|
5
|
-
modbusRtuBusSource,
|
|
6
|
-
modbusRtuSource,
|
|
7
|
-
modbusSource,
|
|
8
|
-
mqttSink
|
|
9
|
-
} from '@yarkivaev/source-to-sink';
|
|
1
|
+
import { clock } from '@yarkivaev/source-to-sink';
|
|
2
|
+
import { modbusRtuBusSource, modbusRtuSource, modbusSource, mqttSink } from '../../streams/index.js';
|
|
3
|
+
import ingestBatch from '../ingestBatch.js';
|
|
10
4
|
import { parseModbusDeviceSpec } from './modbusDeviceSpec.js';
|
|
11
5
|
|
|
12
6
|
/**
|
|
@@ -181,24 +175,28 @@ function buildSources(devices, config, collector, clk) {
|
|
|
181
175
|
export default function modbusMqtt(mqtt, devices, config) {
|
|
182
176
|
assertModbusMqttArgs(mqtt, devices, config);
|
|
183
177
|
const clk = clock();
|
|
184
|
-
const breaker = circuit(config.threshold, config.timeout, clk);
|
|
185
178
|
const sink = mqttSink(mqtt, {
|
|
186
179
|
clientId: config.clientId || 'scada-modbus',
|
|
187
180
|
qos: 1
|
|
188
181
|
});
|
|
189
|
-
const collector =
|
|
182
|
+
const collector = ingestBatch(sink, { ...config, size: config.size ?? 5 });
|
|
190
183
|
const sources = buildSources(devices, config, collector, clk);
|
|
191
184
|
return {
|
|
192
|
-
start() {
|
|
193
|
-
sink.start();
|
|
194
|
-
|
|
195
|
-
source.start();
|
|
185
|
+
async start() {
|
|
186
|
+
await sink.start();
|
|
187
|
+
try {
|
|
188
|
+
await Promise.all(sources.map((source) => { return source.start(); }));
|
|
189
|
+
} catch (error) {
|
|
190
|
+
await Promise.allSettled(sources.map((source) => { return source.stop(); }));
|
|
191
|
+
await sink.stop();
|
|
192
|
+
throw error;
|
|
196
193
|
}
|
|
197
194
|
},
|
|
198
|
-
stop() {
|
|
199
|
-
|
|
200
|
-
source.stop();
|
|
201
|
-
|
|
195
|
+
async stop() {
|
|
196
|
+
try {
|
|
197
|
+
await Promise.all(sources.map((source) => { return source.stop(); }));
|
|
198
|
+
await collector.stop();
|
|
199
|
+
} finally { await sink.stop(); }
|
|
202
200
|
}
|
|
203
201
|
};
|
|
204
202
|
}
|
|
@@ -260,6 +260,10 @@ export default function modbusPublish(mqtt, config) {
|
|
|
260
260
|
return [row.name, plans[index]];
|
|
261
261
|
}));
|
|
262
262
|
return modbusMqtt(mqtt, specs, {
|
|
263
|
+
budget: config.budget,
|
|
264
|
+
maxRecords: config.maxRecords,
|
|
265
|
+
maxBytes: config.maxBytes,
|
|
266
|
+
writeTimeoutMs: config.writeTimeoutMs,
|
|
263
267
|
address: frame.address,
|
|
264
268
|
clientId: config.clientId || 'scada-modbus',
|
|
265
269
|
count: frame.count,
|
|
@@ -1,21 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
circuit,
|
|
4
|
-
clock,
|
|
5
|
-
mqttSource,
|
|
6
|
-
timedBatch
|
|
7
|
-
} from '@yarkivaev/source-to-sink';
|
|
1
|
+
import { mqttSource } from '../../streams/index.js';
|
|
2
|
+
import ingestBatch from '../ingestBatch.js';
|
|
8
3
|
import metricsCodec from './metricsTransformer.js';
|
|
9
4
|
|
|
10
5
|
/**
|
|
11
6
|
* Pipeline for streaming MQTT sensor data to a storage sink.
|
|
12
7
|
*
|
|
13
|
-
* Subscribes to MQTT
|
|
14
|
-
*
|
|
15
|
-
* and
|
|
8
|
+
* Subscribes to MQTT and writes each message before acknowledging it.
|
|
9
|
+
* MQTT.js serializes acknowledgement, so acceptance cannot wait for a larger batch.
|
|
10
|
+
* Bounds queued records and bytes and enforces a durable-write deadline.
|
|
16
11
|
*
|
|
17
12
|
* @example
|
|
18
|
-
* import { clickhouseSink } from '
|
|
13
|
+
* import { clickhouseSink } from '../../streams/index.js';
|
|
19
14
|
* const sink = clickhouseSink('http://localhost:8123', 'scada.metrics');
|
|
20
15
|
* const pipeline = mqttMetrics(
|
|
21
16
|
* 'mqtt://localhost:1883',
|
|
@@ -31,10 +26,10 @@ import metricsCodec from './metricsTransformer.js';
|
|
|
31
26
|
* @param {object} sink - Sink with write(records) method
|
|
32
27
|
* @param {string} topic - MQTT topic pattern to subscribe
|
|
33
28
|
* @param {object} config - Pipeline configuration
|
|
34
|
-
* @param {number} config.size -
|
|
29
|
+
* @param {number} config.size - Legacy option; serial durable acceptance writes one record at a time
|
|
35
30
|
* @param {number} config.interval - Seconds before time-based flush
|
|
36
|
-
* @param {number} config.threshold -
|
|
37
|
-
* @param {number} config.timeout -
|
|
31
|
+
* @param {number} config.threshold - Legacy option (retained for configuration compatibility)
|
|
32
|
+
* @param {number} config.timeout - Legacy option (use writeTimeoutMs for write deadlines)
|
|
38
33
|
* @param {string} [config.clientId] - MQTT client ID for persistent sessions
|
|
39
34
|
* @param {number} [config.sessionExpiryInterval] - Session expiry in seconds (default 3600)
|
|
40
35
|
* @returns {object} Pipeline with start() and stop() methods
|
|
@@ -52,11 +47,12 @@ export default function mqttMetrics(mqtt, sink, topic, config) {
|
|
|
52
47
|
if (!config || typeof config !== 'object') {
|
|
53
48
|
throw new Error('Config must be an object');
|
|
54
49
|
}
|
|
55
|
-
const
|
|
56
|
-
const breaker = circuit(config.threshold, config.timeout, clk);
|
|
57
|
-
const collector = timedBatch(batch(sink, config.size, breaker), config.interval);
|
|
50
|
+
const collector = ingestBatch(sink, { ...config, size: 1 });
|
|
58
51
|
const transformer = metricsCodec(collector);
|
|
59
52
|
const source = mqttSource(mqtt, topic, transformer, {
|
|
53
|
+
budget: config.budget,
|
|
54
|
+
prefetch: config.prefetch,
|
|
55
|
+
maxPacketBytes: config.maxBytes,
|
|
60
56
|
clientId: config.clientId,
|
|
61
57
|
sessionExpiryInterval: config.sessionExpiryInterval
|
|
62
58
|
});
|
|
@@ -65,14 +61,14 @@ export default function mqttMetrics(mqtt, sink, topic, config) {
|
|
|
65
61
|
* Starts the pipeline.
|
|
66
62
|
*/
|
|
67
63
|
start() {
|
|
68
|
-
source.start();
|
|
64
|
+
return source.start();
|
|
69
65
|
},
|
|
70
66
|
/**
|
|
71
67
|
* Stops the pipeline.
|
|
72
68
|
*/
|
|
73
|
-
stop() {
|
|
74
|
-
source.stop();
|
|
75
|
-
collector.stop();
|
|
69
|
+
async stop() {
|
|
70
|
+
await source.stop();
|
|
71
|
+
await collector.stop();
|
|
76
72
|
}
|
|
77
73
|
};
|
|
78
74
|
}
|
|
@@ -70,15 +70,15 @@ export default function registerPublish(plans, origin, collector) {
|
|
|
70
70
|
* @param {Array<number>} registers - uint16 window
|
|
71
71
|
*/
|
|
72
72
|
accept(registers) {
|
|
73
|
-
rows.
|
|
74
|
-
collector.accept({
|
|
73
|
+
return Promise.all(rows.map((item) => {
|
|
74
|
+
return collector.accept({
|
|
75
75
|
payload: JSON.stringify({
|
|
76
76
|
ts: Date.now(),
|
|
77
77
|
value: decodePoint(item.point, origin, registers)
|
|
78
78
|
}),
|
|
79
79
|
topic: item.topic
|
|
80
80
|
});
|
|
81
|
-
});
|
|
81
|
+
}));
|
|
82
82
|
}
|
|
83
83
|
};
|
|
84
84
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import stompOptions from './stompOptions.js';
|
|
2
|
+
import { stompSource } from '../../streams/index.js';
|
|
2
3
|
import alertCodec from '../codecs/alertCodec.js';
|
|
3
4
|
import alertSink from '../sinks/alertSink.js';
|
|
4
5
|
|
|
@@ -29,20 +30,20 @@ export default function alertPipeline(stomp, pool, config) {
|
|
|
29
30
|
const sink = alertSink(pool);
|
|
30
31
|
const codec = alertCodec(sink, config.translations || {});
|
|
31
32
|
const source = stompSource(stomp, '/exchange/scada.alerts', codec,
|
|
32
|
-
|
|
33
|
+
stompOptions(config));
|
|
33
34
|
return {
|
|
34
35
|
/**
|
|
35
36
|
* Starts the pipeline.
|
|
36
37
|
*/
|
|
37
38
|
start() {
|
|
38
|
-
source.start();
|
|
39
|
+
return source.start();
|
|
39
40
|
},
|
|
40
41
|
/**
|
|
41
42
|
* Stops the pipeline.
|
|
42
43
|
*/
|
|
43
|
-
stop() {
|
|
44
|
-
source.stop();
|
|
45
|
-
sink.stop();
|
|
44
|
+
async stop() {
|
|
45
|
+
await source.stop();
|
|
46
|
+
await sink.stop();
|
|
46
47
|
}
|
|
47
48
|
};
|
|
48
49
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import stompOptions from './stompOptions.js';
|
|
2
|
+
import { stompSource } from '../../streams/index.js';
|
|
2
3
|
import userDecisionCodec from '../codecs/userDecisionCodec.js';
|
|
3
4
|
import userDecisionSink from '../sinks/userDecisionSink.js';
|
|
4
5
|
|
|
@@ -26,20 +27,20 @@ export default function decisionPipeline(stomp, pool, config) {
|
|
|
26
27
|
const sink = userDecisionSink(pool);
|
|
27
28
|
const codec = userDecisionCodec(sink);
|
|
28
29
|
const source = stompSource(stomp, '/exchange/scada.user_decisions', codec,
|
|
29
|
-
|
|
30
|
+
stompOptions(config));
|
|
30
31
|
return {
|
|
31
32
|
/**
|
|
32
33
|
* Starts consuming user decisions from RabbitMQ.
|
|
33
34
|
*/
|
|
34
35
|
start() {
|
|
35
|
-
source.start();
|
|
36
|
+
return source.start();
|
|
36
37
|
},
|
|
37
38
|
/**
|
|
38
39
|
* Stops consuming and releases resources.
|
|
39
40
|
*/
|
|
40
|
-
stop() {
|
|
41
|
-
source.stop();
|
|
42
|
-
pool.end();
|
|
41
|
+
async stop() {
|
|
42
|
+
await source.stop();
|
|
43
|
+
await pool.end();
|
|
43
44
|
}
|
|
44
45
|
};
|
|
45
46
|
}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
stompSource
|
|
4
|
-
} from '@yarkivaev/source-to-sink';
|
|
1
|
+
import stompOptions from './stompOptions.js';
|
|
2
|
+
import { postgresSink, stompSource } from '../../streams/index.js';
|
|
5
3
|
import segmentDispatch from '../../../domain/segment/dispatch.js';
|
|
6
4
|
import segmentCoalesce from '../../../domain/segment/segmentCoalesce.js';
|
|
7
5
|
import silenceBudget from '../../../domain/segment/silenceBudget.js';
|
|
@@ -64,14 +62,7 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
|
|
|
64
62
|
interval: config.interval
|
|
65
63
|
});
|
|
66
64
|
const destination = config.segmentsDestination || segmentsIngestDestination;
|
|
67
|
-
const source = stompSource(stomp, destination, coalesce, {
|
|
68
|
-
login: config.login,
|
|
69
|
-
passcode: config.passcode,
|
|
70
|
-
host: config.host,
|
|
71
|
-
serial: false,
|
|
72
|
-
manualAck: true,
|
|
73
|
-
prefetch: config.prefetch
|
|
74
|
-
});
|
|
65
|
+
const source = stompSource(stomp, destination, coalesce, stompOptions(config, { serial: false, manualAck: true }));
|
|
75
66
|
const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
|
|
76
67
|
intervalMs: (config.poll || 5) * 1000
|
|
77
68
|
});
|
|
@@ -82,13 +73,13 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
|
|
|
82
73
|
void repair.catch((error) => {
|
|
83
74
|
processingErrorLog('abut_segments', error, { budgetSeconds: budget });
|
|
84
75
|
});
|
|
85
|
-
source.start();
|
|
76
|
+
await source.start();
|
|
86
77
|
await silence.start();
|
|
87
78
|
},
|
|
88
|
-
stop() {
|
|
89
|
-
silence.stop();
|
|
90
|
-
source.stop();
|
|
91
|
-
pool.end();
|
|
79
|
+
async stop() {
|
|
80
|
+
await silence.stop();
|
|
81
|
+
await source.stop();
|
|
82
|
+
await pool.end();
|
|
92
83
|
}
|
|
93
84
|
};
|
|
94
85
|
}
|