@yarkivaev/scada 2.3.115 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -0
- package/charts/scada/templates/processes.yaml +22 -0
- package/charts/scada/values.schema.json +12 -0
- package/db/migrations/V0009__collector_task_state.sql +6 -0
- package/package.json +14 -5
- package/src/application/deploy/collectorProcesses.js +3 -2
- package/src/application/deploy/readDocument.js +4 -1
- package/src/application/plantServer.js +1 -1
- package/src/application/runtime/additionalAdapters.js +139 -0
- package/src/application/runtime/assignment.js +30 -0
- package/src/application/runtime/budgetedSink.js +17 -0
- package/src/application/runtime/collectorSignals.js +14 -0
- package/src/application/runtime/compositeLifecycle.js +49 -0
- package/src/application/runtime/createRegistry.js +20 -0
- package/src/application/runtime/deadline.js +8 -0
- package/src/application/runtime/index.js +10 -0
- package/src/application/runtime/manifest.js +38 -0
- package/src/application/runtime/pipelineAdapters.js +60 -0
- package/src/application/runtime/resolveSecrets.js +12 -0
- package/src/application/runtime/resourceClaims.js +21 -0
- package/src/application/runtime/runCollector.js +50 -0
- package/src/application/runtime/runtime.js +86 -0
- package/src/application/runtime/runtimeHealth.js +29 -0
- package/src/application/runtime/runtimeServices.js +15 -0
- package/src/application/runtime/stateDocument.js +22 -0
- package/src/application/runtime/streamPorts.js +42 -0
- package/src/application/runtime/streamTask.js +105 -0
- package/src/application/runtime/taskBudget.js +31 -0
- package/src/application/runtime/taskContext.js +18 -0
- package/src/application/runtime/taskQueue.js +21 -0
- package/src/application/runtime/taskSlot.js +46 -0
- package/src/application/runtime/taskState.js +73 -0
- package/src/application/siteCollection.js +14 -0
- package/src/application/siteServer.js +7 -6
- package/src/application/siteTelemetry.js +1 -1
- package/src/application/supervisorSink.js +9 -8
- package/src/application/sync/siteSyncBind.js +1 -1
- package/src/application/sync/siteSyncTargets.js +1 -1
- package/src/bin/collector.js +7 -0
- package/src/bin/modbus-to-mqtt.js +11 -9
- package/src/infrastructure/deploy/executeRelease.js +45 -3
- package/src/infrastructure/ingest/activity/activityTracking.js +15 -21
- package/src/infrastructure/ingest/activity/activityTransformer.js +16 -15
- package/src/infrastructure/ingest/durableConsume.js +2 -169
- package/src/infrastructure/ingest/ingestBatch.js +13 -0
- package/src/infrastructure/ingest/modbus/mx210Tcp.js +4 -4
- package/src/infrastructure/ingest/modbus/silentStreams.js +39 -16
- package/src/infrastructure/ingest/mqtt/metricsTransformer.js +4 -5
- package/src/infrastructure/ingest/mqtt/modbusMqtt.js +17 -19
- package/src/infrastructure/ingest/mqtt/modbusPublish.js +4 -0
- package/src/infrastructure/ingest/mqtt/mqttMetrics.js +17 -21
- package/src/infrastructure/ingest/mqtt/registerPublish.js +3 -3
- package/src/infrastructure/ingest/pipelines/alertPipeline.js +7 -6
- package/src/infrastructure/ingest/pipelines/decisionPipeline.js +7 -6
- package/src/infrastructure/ingest/pipelines/segmentPipeline.js +8 -17
- package/src/infrastructure/ingest/pipelines/stompOptions.js +7 -0
- package/src/infrastructure/ingest/silentOpenWatch.js +5 -42
- package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js +16 -28
- package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js +17 -22
- package/src/infrastructure/ingest/telemetry/foldingSink.js +5 -2
- package/src/infrastructure/messaging/stomp/segmentRetags.js +1 -1
- package/src/infrastructure/messaging/stomp/userDecisions.js +1 -1
- package/src/infrastructure/opcua/certificates.js +11 -0
- package/src/infrastructure/opcua/config.js +38 -0
- package/src/infrastructure/opcua/delivery.js +38 -0
- package/src/infrastructure/opcua/endpoint.js +90 -0
- package/src/infrastructure/opcua/index.js +10 -0
- package/src/infrastructure/opcua/session.js +70 -0
- package/src/infrastructure/opcua/source.js +38 -0
- package/src/infrastructure/opcua/subscriptionHealth.js +28 -0
- package/src/infrastructure/operators/operatorsSync.js +5 -29
- package/src/infrastructure/persistence/clickhouse/streamHub.js +21 -15
- package/src/infrastructure/persistence/metricsSensor.js +9 -6
- package/src/infrastructure/persistence/runtime/taskStatePg.js +24 -0
- package/src/infrastructure/streams/README.md +291 -0
- package/src/infrastructure/streams/abortable.js +14 -0
- package/src/infrastructure/streams/amqpSink/publication.js +46 -0
- package/src/infrastructure/streams/amqpSink/session.js +96 -0
- package/src/infrastructure/streams/amqpSink.js +47 -0
- package/src/infrastructure/streams/amqpSource/session.js +127 -0
- package/src/infrastructure/streams/amqpSource.js +97 -0
- package/src/infrastructure/streams/brokerClose.js +19 -0
- package/src/infrastructure/streams/budgetAccept.js +6 -0
- package/src/infrastructure/streams/clickhouseSink.js +48 -0
- package/src/infrastructure/streams/deadline.js +1 -0
- package/src/infrastructure/streams/follow.js +41 -0
- package/src/infrastructure/streams/index.js +15 -0
- package/src/infrastructure/streams/lokiSource.js +88 -0
- package/src/infrastructure/streams/modbusConnection.js +173 -0
- package/src/infrastructure/streams/modbusRtuBusSource.js +124 -0
- package/src/infrastructure/streams/modbusSource.js +185 -0
- package/src/infrastructure/streams/mqttSink/session.js +51 -0
- package/src/infrastructure/streams/mqttSink.js +54 -0
- package/src/infrastructure/streams/mqttSource.js +66 -0
- package/src/infrastructure/streams/pollingSource.js +1 -0
- package/src/infrastructure/streams/positive.js +5 -0
- package/src/infrastructure/streams/postgresSink.js +124 -0
- package/src/infrastructure/streams/serialReceive.js +80 -0
- package/src/infrastructure/streams/serialSource.js +87 -0
- package/src/infrastructure/streams/sqliteSink.js +55 -0
- package/src/infrastructure/streams/stompBody.js +46 -0
- package/src/infrastructure/streams/stompSend.js +89 -0
- package/src/infrastructure/streams/stompSource.js +201 -0
- package/src/infrastructure/sync/operationSyncIngest.js +4 -16
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import pollingSource from './pollingSource.js';
|
|
2
|
+
import deadline from './deadline.js';
|
|
3
|
+
import abortable from './abortable.js';
|
|
4
|
+
import { attempt, create, drop, idle, opening, resume } from './modbusConnection.js';
|
|
5
|
+
|
|
6
|
+
function read(bag, address, count) {
|
|
7
|
+
return async function fetch(since, until, { signal } = {}) {
|
|
8
|
+
const { client } = bag;
|
|
9
|
+
try {
|
|
10
|
+
const result = await abortable(client.readHoldingRegisters(address, count), signal, () => { drop(client); });
|
|
11
|
+
return [result.data];
|
|
12
|
+
} catch (err) {
|
|
13
|
+
if (bag.client === client) { bag.lost(); }
|
|
14
|
+
throw err;
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Shared Modbus polling lifecycle for TCP and RTU transports.
|
|
21
|
+
*
|
|
22
|
+
* @param {object} options - connect, target, address, count, interval, collector, clk, client, log
|
|
23
|
+
* @returns {object} Source with start() and stop() methods
|
|
24
|
+
*/
|
|
25
|
+
function modbusPollingSource(options) {
|
|
26
|
+
const { address, count, interval, collector, clk } = options;
|
|
27
|
+
const bag = {
|
|
28
|
+
...options,
|
|
29
|
+
client: undefined,
|
|
30
|
+
state: idle(),
|
|
31
|
+
epoch: 0,
|
|
32
|
+
maxDelay: 300,
|
|
33
|
+
delay: interval
|
|
34
|
+
};
|
|
35
|
+
bag.lost = function lost() {
|
|
36
|
+
resume(bag);
|
|
37
|
+
};
|
|
38
|
+
bag.source = pollingSource(read(bag, address, count), interval, collector, clk);
|
|
39
|
+
return {
|
|
40
|
+
/**
|
|
41
|
+
* Connects to the Modbus device and starts polling.
|
|
42
|
+
*/
|
|
43
|
+
start() {
|
|
44
|
+
if (bag.fatal) { return Promise.reject(bag.fatal); }
|
|
45
|
+
if (bag.state.connected() || bag.state.retrying() || bag.state.opening()) {
|
|
46
|
+
return Promise.resolve();
|
|
47
|
+
}
|
|
48
|
+
const token = bag.epoch;
|
|
49
|
+
bag.state = opening();
|
|
50
|
+
return Promise.resolve(bag.stopping).then(() => { if (token === bag.epoch) { attempt(bag); } });
|
|
51
|
+
},
|
|
52
|
+
/**
|
|
53
|
+
* Stops polling and disconnects from the Modbus device.
|
|
54
|
+
*/
|
|
55
|
+
stop() {
|
|
56
|
+
bag.epoch += 1;
|
|
57
|
+
const drained = bag.source.stop();
|
|
58
|
+
if (bag.state.retrying()) {
|
|
59
|
+
bag.state.cancel();
|
|
60
|
+
}
|
|
61
|
+
const { client } = bag;
|
|
62
|
+
bag.client = undefined;
|
|
63
|
+
bag.state = idle();
|
|
64
|
+
drop(client);
|
|
65
|
+
bag.stopping = deadline(Promise.all([drained, bag.opened]), 5000, 'Modbus stop', (error) => { bag.fatal = error; }).then(() => {
|
|
66
|
+
if (bag.fatal) { throw bag.fatal; }
|
|
67
|
+
});
|
|
68
|
+
return bag.stopping;
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Modbus TCP polling source for reading holding registers.
|
|
75
|
+
*
|
|
76
|
+
* Connects to a Modbus TCP device and polls holding registers
|
|
77
|
+
* at a specified interval. Forwards register arrays to the
|
|
78
|
+
* collector via pollingSource.
|
|
79
|
+
*
|
|
80
|
+
* @example
|
|
81
|
+
* const source = modbusSource('192.168.2.148', 502, 4000, 14, 5, collector, clock);
|
|
82
|
+
* source.start();
|
|
83
|
+
*
|
|
84
|
+
* @param {string} host - Modbus TCP host address
|
|
85
|
+
* @param {number} port - Modbus TCP port
|
|
86
|
+
* @param {number} address - Starting register address
|
|
87
|
+
* @param {number} count - Number of registers to read
|
|
88
|
+
* @param {number} interval - Polling interval in seconds
|
|
89
|
+
* @param {object} collector - Collector with accept() method
|
|
90
|
+
* @param {object} clk - Clock with millis() method
|
|
91
|
+
* @param {object} [log] - Logger with error() method for connection failures
|
|
92
|
+
* @returns {object} Source with start() and stop() methods
|
|
93
|
+
*/
|
|
94
|
+
// Preserve the established positional adapter API.
|
|
95
|
+
// eslint-disable-next-line max-params
|
|
96
|
+
export default function modbusSource(host, port, address, count, interval, collector, clk, log = console) {
|
|
97
|
+
if (typeof host !== 'string' || host.length === 0) {
|
|
98
|
+
throw new Error('Host must be a non-empty string');
|
|
99
|
+
}
|
|
100
|
+
if (typeof port !== 'number' || port <= 0) {
|
|
101
|
+
throw new Error(`Port must be a positive number, got: ${port}`);
|
|
102
|
+
}
|
|
103
|
+
if (typeof address !== 'number' || address < 0) {
|
|
104
|
+
throw new Error(`Address must be a non-negative number, got: ${address}`);
|
|
105
|
+
}
|
|
106
|
+
if (typeof count !== 'number' || count <= 0) {
|
|
107
|
+
throw new Error(`Count must be a positive number, got: ${count}`);
|
|
108
|
+
}
|
|
109
|
+
function connect(modbusClient) {
|
|
110
|
+
return modbusClient.connectTCP(host, { port });
|
|
111
|
+
}
|
|
112
|
+
return modbusPollingSource({
|
|
113
|
+
create,
|
|
114
|
+
connect,
|
|
115
|
+
target: `${host}:${port}`,
|
|
116
|
+
address,
|
|
117
|
+
count,
|
|
118
|
+
interval,
|
|
119
|
+
collector,
|
|
120
|
+
clk,
|
|
121
|
+
log
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Modbus RTU polling source for reading holding registers over serial.
|
|
127
|
+
*
|
|
128
|
+
* Connects to a Modbus RTU device on a serial port and polls holding registers
|
|
129
|
+
* at a specified interval. Forwards register arrays to the collector via pollingSource.
|
|
130
|
+
*
|
|
131
|
+
* @example
|
|
132
|
+
* const source = modbusRtuSource('/dev/ttyUSB0', { baudRate: 9600, slaveId: 1 }, 4000, 44, 5, collector, clock);
|
|
133
|
+
* source.start();
|
|
134
|
+
*
|
|
135
|
+
* @param {string} path - Serial device path (e.g. /dev/ttyUSB0)
|
|
136
|
+
* @param {object} serial - baudRate, dataBits, stopBits, parity, slaveId
|
|
137
|
+
* @param {number} address - Starting register address
|
|
138
|
+
* @param {number} count - Number of registers to read
|
|
139
|
+
* @param {number} interval - Polling interval in seconds
|
|
140
|
+
* @param {object} collector - Collector with accept() method
|
|
141
|
+
* @param {object} clk - Clock with millis() method
|
|
142
|
+
* @param {object} [log] - Logger with error() method for connection failures
|
|
143
|
+
* @returns {object} Source with start() and stop() methods
|
|
144
|
+
*/
|
|
145
|
+
// Preserve the established positional adapter API.
|
|
146
|
+
// eslint-disable-next-line max-params
|
|
147
|
+
export function modbusRtuSource(path, serial, address, count, interval, collector, clk, log = console) {
|
|
148
|
+
if (typeof path !== 'string' || path.length === 0) {
|
|
149
|
+
throw new Error('Serial path must be a non-empty string');
|
|
150
|
+
}
|
|
151
|
+
if (!serial || typeof serial !== 'object') {
|
|
152
|
+
throw new Error('Serial config must be an object');
|
|
153
|
+
}
|
|
154
|
+
if (typeof serial.baudRate !== 'number' || serial.baudRate <= 0) {
|
|
155
|
+
throw new Error(`Baud rate must be a positive number, got: ${serial.baudRate}`);
|
|
156
|
+
}
|
|
157
|
+
if (typeof address !== 'number' || address < 0) {
|
|
158
|
+
throw new Error(`Address must be a non-negative number, got: ${address}`);
|
|
159
|
+
}
|
|
160
|
+
if (typeof count !== 'number' || count <= 0) {
|
|
161
|
+
throw new Error(`Count must be a positive number, got: ${count}`);
|
|
162
|
+
}
|
|
163
|
+
const slaveId = serial.slaveId || 1;
|
|
164
|
+
function connect(modbusClient) {
|
|
165
|
+
return modbusClient.connectRTUBuffered(path, {
|
|
166
|
+
baudRate: serial.baudRate,
|
|
167
|
+
dataBits: serial.dataBits || 8,
|
|
168
|
+
stopBits: serial.stopBits || 1,
|
|
169
|
+
parity: serial.parity || 'none'
|
|
170
|
+
}).then(() => {
|
|
171
|
+
modbusClient.setID(slaveId);
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
return modbusPollingSource({
|
|
175
|
+
create,
|
|
176
|
+
connect,
|
|
177
|
+
target: path,
|
|
178
|
+
address,
|
|
179
|
+
count,
|
|
180
|
+
interval,
|
|
181
|
+
collector,
|
|
182
|
+
clk,
|
|
183
|
+
log
|
|
184
|
+
});
|
|
185
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import deadline from '../deadline.js';
|
|
2
|
+
|
|
3
|
+
export function terminate(live, error) {
|
|
4
|
+
if (live.dead) { return live.closing; }
|
|
5
|
+
live.dead = true;
|
|
6
|
+
live.ready = false;
|
|
7
|
+
live.pending.forEach((settle) => { settle(error || new Error('MQTT connection closed')); });
|
|
8
|
+
live.closing = deadline(new Promise((resolve) => {
|
|
9
|
+
live.client.end(true, {}, resolve);
|
|
10
|
+
}), live.options.stopTimeoutMs, 'MQTT disconnect');
|
|
11
|
+
// An event callback cannot surface a rejected shutdown. stop() still returns it.
|
|
12
|
+
void live.closing.catch(() => { return undefined; });
|
|
13
|
+
return live.closing;
|
|
14
|
+
}
|
|
15
|
+
export function connect(live) {
|
|
16
|
+
return deadline(new Promise((resolve, reject) => {
|
|
17
|
+
function fail(error) { reject(error); void terminate(live, error); }
|
|
18
|
+
live.client.on('error', fail);
|
|
19
|
+
live.client.on('close', () => { fail(new Error('MQTT connection closed')); });
|
|
20
|
+
live.client.on('offline', () => { fail(new Error('MQTT connection offline')); });
|
|
21
|
+
live.client.on('connect', () => {
|
|
22
|
+
if (!live.dead) { live.ready = true; resolve(); }
|
|
23
|
+
});
|
|
24
|
+
live.pending.add(reject);
|
|
25
|
+
}), live.options.connectTimeoutMs, 'MQTT connect', (error) => { void terminate(live, error); });
|
|
26
|
+
}
|
|
27
|
+
export function publish(live, record, signal) {
|
|
28
|
+
return new Promise((resolve, reject) => {
|
|
29
|
+
let done = false;
|
|
30
|
+
const timer = { handle: null };
|
|
31
|
+
function abort() { void terminate(live, signal.reason || new Error('MQTT publish aborted')); }
|
|
32
|
+
function settle(error) {
|
|
33
|
+
if (done) { return; }
|
|
34
|
+
done = true;
|
|
35
|
+
clearTimeout(timer.handle);
|
|
36
|
+
signal?.removeEventListener('abort', abort);
|
|
37
|
+
live.pending.delete(settle);
|
|
38
|
+
live.count -= 1;
|
|
39
|
+
if (error) { reject(error); } else { resolve(); }
|
|
40
|
+
}
|
|
41
|
+
timer.handle = setTimeout(() => { void terminate(live, new Error('MQTT confirmation timed out; outcome uncertain')); }, live.options.confirmTimeoutMs);
|
|
42
|
+
live.pending.add(settle);
|
|
43
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
44
|
+
if (signal?.aborted) { abort(); return; }
|
|
45
|
+
try { live.client.publish(record.topic, record.payload, { qos: live.options.qos }, (error) => {
|
|
46
|
+
settle(error);
|
|
47
|
+
if (error) { void terminate(live, error); }
|
|
48
|
+
}); }
|
|
49
|
+
catch (error) { void terminate(live, error); }
|
|
50
|
+
});
|
|
51
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import mqtt from 'mqtt';
|
|
2
|
+
import positive from './positive.js';
|
|
3
|
+
import { connect, publish, terminate } from './mqttSink/session.js';
|
|
4
|
+
|
|
5
|
+
function settings(options) {
|
|
6
|
+
const config = { qos: 0, maxPending: 1024, maxPendingBytes: 1048576, connectTimeoutMs: 5000, confirmTimeoutMs: 10000, stopTimeoutMs: 5000, ...options };
|
|
7
|
+
['maxPending', 'maxPendingBytes', 'connectTimeoutMs', 'confirmTimeoutMs', 'stopTimeoutMs'].forEach((key) => { positive(config[key], key); });
|
|
8
|
+
if (!Number.isSafeInteger(config.maxPending)) { throw new TypeError('maxPending must be a safe integer'); }
|
|
9
|
+
if (![0, 1, 2].includes(config.qos)) { throw new TypeError('qos must be 0, 1, or 2'); }
|
|
10
|
+
return config;
|
|
11
|
+
}
|
|
12
|
+
function write(live, records, signal) {
|
|
13
|
+
if (!live?.ready || live.dead) { throw new Error('Sink is not connected'); }
|
|
14
|
+
if (!Array.isArray(records) || records.some((record) => {
|
|
15
|
+
return !record || typeof record.topic !== 'string' || !record.topic || !(typeof record.payload === 'string' || Buffer.isBuffer(record.payload));
|
|
16
|
+
})) { return Promise.reject(new TypeError('MQTT records require a topic and string/Buffer payload')); }
|
|
17
|
+
if (signal?.aborted) { return Promise.reject(signal.reason || new Error('MQTT publish aborted')); }
|
|
18
|
+
const bytes = records.reduce((total, record) => { return total + Buffer.byteLength(record.payload) + Buffer.byteLength(record.topic); }, 0);
|
|
19
|
+
if (live.bytes + bytes > live.options.maxPendingBytes || live.count + records.length > live.options.maxPending) { return Promise.reject(new Error('MQTT pending capacity exceeded')); }
|
|
20
|
+
live.count += records.length;
|
|
21
|
+
live.bytes += bytes;
|
|
22
|
+
return Promise.all(records.map((record) => {
|
|
23
|
+
if (live.dead) { live.count -= 1; return Promise.reject(new Error('MQTT connection closed')); }
|
|
24
|
+
return publish(live, record, signal);
|
|
25
|
+
})).finally(() => { live.bytes -= bytes; });
|
|
26
|
+
}
|
|
27
|
+
/** start awaits CONNACK; write awaits publish completion (broker confirmation at QoS 1/2).
|
|
28
|
+
* QoS 0 only confirms local handoff. No automatic replay of uncertain writes.
|
|
29
|
+
*/
|
|
30
|
+
export default function mqttSink(url, options = {}) {
|
|
31
|
+
if (typeof url !== 'string' || !url) { throw new Error('URL must be a non-empty string'); }
|
|
32
|
+
const config = settings(options);
|
|
33
|
+
let live;
|
|
34
|
+
return {
|
|
35
|
+
start() {
|
|
36
|
+
if (live && !live.dead) { return live.opening; }
|
|
37
|
+
if (live?.dead) { return Promise.reject(new Error('MQTT session retired; create a new sink after stop')); }
|
|
38
|
+
const client = (options.connect || mqtt.connect)(url, {
|
|
39
|
+
...options.connectionOptions, clientId: options.clientId, protocolVersion: 5,
|
|
40
|
+
reconnectPeriod: 0, queueQoSZero: false, connectTimeout: config.connectTimeoutMs
|
|
41
|
+
});
|
|
42
|
+
live = { client, options: config, pending: new Set(), count: 0, bytes: 0, ready: false, dead: false };
|
|
43
|
+
live.opening = connect(live);
|
|
44
|
+
return live.opening;
|
|
45
|
+
},
|
|
46
|
+
write(records, { signal } = {}) { return write(live, records, signal); },
|
|
47
|
+
stop() {
|
|
48
|
+
const stopped = live;
|
|
49
|
+
return (stopped ? terminate(stopped, new Error('MQTT sink stopped')) : Promise.resolve()).then(() => {
|
|
50
|
+
if (live === stopped) { live = undefined; }
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import mqtt from 'mqtt';
|
|
2
|
+
import positive from './positive.js';
|
|
3
|
+
import deadline from './deadline.js';
|
|
4
|
+
import budgetAccept from './budgetAccept.js';
|
|
5
|
+
|
|
6
|
+
function report(options, error) {
|
|
7
|
+
try { options.onError?.(error); } catch {
|
|
8
|
+
// Diagnostics cannot break transport callbacks.
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
function bind(live, collector, options) {
|
|
12
|
+
live.client.on('error', (error) => { report(options, error); });
|
|
13
|
+
// MQTT.js serializes handleMessage and sends PUBACK/PUBCOMP only on successful completion.
|
|
14
|
+
live.client.handleMessage = (packet, callback) => {
|
|
15
|
+
const accepted = Promise.resolve().then(() => {
|
|
16
|
+
if (!live.active) { throw new Error('MQTT source stopped'); }
|
|
17
|
+
return budgetAccept(collector, { topic: packet.topic, payload: packet.payload.toString() }, packet.payload.length, options, { signal: live.controller.signal });
|
|
18
|
+
});
|
|
19
|
+
live.pending.add(accepted);
|
|
20
|
+
void accepted.finally(() => { live.pending.delete(accepted); }).catch(() => { return undefined; });
|
|
21
|
+
deadline(accepted, options.acceptTimeoutMs, 'MQTT acceptance', (error) => {
|
|
22
|
+
live.active = false; live.controller.abort(error); live.client.end(true);
|
|
23
|
+
}).then(() => {
|
|
24
|
+
callback(live.active ? undefined : new Error('MQTT source stopped'));
|
|
25
|
+
}, (error) => { callback(error); report(options, error); });
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function open(url, list, collector, options) {
|
|
29
|
+
const client = (options.connect || mqtt.connect)(url, {
|
|
30
|
+
...options.connectionOptions, clientId: options.clientId, clean: !options.clientId, protocolVersion: 5,
|
|
31
|
+
properties: { sessionExpiryInterval: options.clientId ? options.sessionExpiryInterval ?? 3600 : 0,
|
|
32
|
+
receiveMaximum: options.prefetch ?? 32, maximumPacketSize: options.maxPacketBytes ?? 1048576 }
|
|
33
|
+
});
|
|
34
|
+
const live = { client, active: true, pending: new Set(), controller: new AbortController() };
|
|
35
|
+
bind(live, collector, options);
|
|
36
|
+
client.on('connect', () => {
|
|
37
|
+
if (live.active) { client.subscribe(list, { qos: options.qos ?? (options.clientId ? 1 : 0) }); }
|
|
38
|
+
});
|
|
39
|
+
return live;
|
|
40
|
+
}
|
|
41
|
+
/** ACK follows asynchronous acceptance, not merely receipt. Acceptance is not necessarily durable. */
|
|
42
|
+
export default function mqttSource(url, topics, collector, options = {}) {
|
|
43
|
+
if (typeof url !== 'string' || !url) { throw new Error('URL must be a non-empty string'); }
|
|
44
|
+
if (typeof topics !== 'string' || !topics.trim()) { throw new Error('Topic must be a non-empty string'); }
|
|
45
|
+
if (typeof collector?.accept !== 'function') { throw new Error('Collector must have an accept() method'); }
|
|
46
|
+
const config = { acceptTimeoutMs: 10000, stopTimeoutMs: 5000, ...options };
|
|
47
|
+
positive(config.acceptTimeoutMs, 'acceptTimeoutMs');
|
|
48
|
+
positive(config.stopTimeoutMs, 'stopTimeoutMs');
|
|
49
|
+
const list = topics.split(',').map((topic) => { return topic.trim(); }).filter(Boolean);
|
|
50
|
+
let live;
|
|
51
|
+
return {
|
|
52
|
+
start() {
|
|
53
|
+
if (!live) { live = open(url, list, collector, config); }
|
|
54
|
+
},
|
|
55
|
+
stop() {
|
|
56
|
+
if (!live) { return Promise.resolve(); }
|
|
57
|
+
const stopped = live;
|
|
58
|
+
stopped.active = false;
|
|
59
|
+
stopped.controller.abort(new Error('MQTT source stopped'));
|
|
60
|
+
const closing = new Promise((resolve) => { stopped.client.end(true, {}, resolve); });
|
|
61
|
+
return deadline(Promise.all([closing, ...stopped.pending]), config.stopTimeoutMs, 'MQTT stop').then(() => {
|
|
62
|
+
if (live === stopped) { live = undefined; }
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { pollingSource as default } from '@yarkivaev/source-to-sink';
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import pg from 'pg';
|
|
2
|
+
import deadline from './deadline.js';
|
|
3
|
+
import positive from './positive.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Removes duplicate records by conflict key within a batch.
|
|
7
|
+
*
|
|
8
|
+
* PostgreSQL rejects INSERT batches where ON CONFLICT would update
|
|
9
|
+
* the same row twice. This keeps the last occurrence for each
|
|
10
|
+
* unique combination of conflict column values.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* deduplicate(
|
|
14
|
+
* [{ machine: 'a', start: 1, name: 'pending' },
|
|
15
|
+
* { machine: 'a', start: 1, name: 'completed' }],
|
|
16
|
+
* ['machine', 'start']
|
|
17
|
+
* );
|
|
18
|
+
* // => [{ machine: 'a', start: 1, name: 'completed' }]
|
|
19
|
+
*
|
|
20
|
+
* @param {Array} records - Array of record objects
|
|
21
|
+
* @param {Array<string>} keys - Conflict column names
|
|
22
|
+
* @returns {Array} Deduplicated records
|
|
23
|
+
*/
|
|
24
|
+
export function deduplicate(records, keys) {
|
|
25
|
+
if (keys.length === 0) {return records;}
|
|
26
|
+
const seen = new Map();
|
|
27
|
+
for (const record of records) {
|
|
28
|
+
const key = keys.map(k => {return record[k]}).join('\0');
|
|
29
|
+
seen.set(key, record);
|
|
30
|
+
}
|
|
31
|
+
return Array.from(seen.values());
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Builds the ON CONFLICT SQL suffix from options.
|
|
36
|
+
*
|
|
37
|
+
* @param {object} options - Sink options with conflict, update, and optional updateWhere
|
|
38
|
+
* @returns {string} SQL suffix or empty string
|
|
39
|
+
*/
|
|
40
|
+
function buildSuffix(options) {
|
|
41
|
+
if (!Array.isArray(options.conflict) || options.conflict.length === 0) {return '';}
|
|
42
|
+
const cols = options.conflict.join(', ');
|
|
43
|
+
if (Array.isArray(options.update) && options.update.length > 0) {
|
|
44
|
+
const sets = options.update.map((col) => {return `${col} = EXCLUDED.${col}`}).join(', ');
|
|
45
|
+
const guard = typeof options.updateWhere === 'string' && options.updateWhere.length > 0
|
|
46
|
+
? ` WHERE ${options.updateWhere}`
|
|
47
|
+
: '';
|
|
48
|
+
return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}${guard}`;
|
|
49
|
+
}
|
|
50
|
+
return ` ON CONFLICT (${cols}) DO NOTHING`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* PostgreSQL sink for batch record insertion.
|
|
55
|
+
*
|
|
56
|
+
* Creates a pg.Pool internally unless `options.pool` is supplied.
|
|
57
|
+
* Supports optional conflict resolution via ON CONFLICT DO NOTHING or
|
|
58
|
+
* ON CONFLICT DO UPDATE SET. `write` accepts an optional queryable
|
|
59
|
+
* client so callers can run inserts inside an existing transaction.
|
|
60
|
+
*
|
|
61
|
+
* @example
|
|
62
|
+
* const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
|
|
63
|
+
* await sink.write([{ ts: Date.now(), value: 42 }]);
|
|
64
|
+
*
|
|
65
|
+
* @example
|
|
66
|
+
* const sink = postgresSink(url, 'segments', cols, { pool, conflict: ['machine', 'start_time'] });
|
|
67
|
+
* await sink.write([{ machine: 'a', name: 'on', start_time: '2024-01-01' }], client);
|
|
68
|
+
*
|
|
69
|
+
* @param {string} url - PostgreSQL URL (e.g., 'postgresql://localhost:5432/db')
|
|
70
|
+
* @param {string} table - Target table name
|
|
71
|
+
* @param {Array<string>} columns - Column names for insertion
|
|
72
|
+
* @param {object} [options] - Optional configuration
|
|
73
|
+
* @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
|
|
74
|
+
* @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
|
|
75
|
+
* @param {string} [options.updateWhere] - Optional WHERE guard on DO UPDATE
|
|
76
|
+
* @param {object} [options.pool] - Existing pg Pool to reuse
|
|
77
|
+
* @returns {object} Sink with write(records, client?) method
|
|
78
|
+
*/
|
|
79
|
+
export default function postgresSink(url, table, columns, options = {}) {
|
|
80
|
+
if (typeof url !== 'string' || url.length === 0) {
|
|
81
|
+
throw new Error('URL must be a non-empty string');
|
|
82
|
+
}
|
|
83
|
+
if (typeof table !== 'string' || table.length === 0) {
|
|
84
|
+
throw new Error('Table must be a non-empty string');
|
|
85
|
+
}
|
|
86
|
+
if (!Array.isArray(columns) || columns.length === 0) {
|
|
87
|
+
throw new Error('Columns must be a non-empty array');
|
|
88
|
+
}
|
|
89
|
+
positive(options.stopTimeoutMs ?? 5000, 'stopTimeoutMs');
|
|
90
|
+
const pool = options.pool || (options.createPool || ((config) => { return new pg.Pool(config); }))({ connectionString: url });
|
|
91
|
+
let stopping;
|
|
92
|
+
const suffix = buildSuffix(options);
|
|
93
|
+
const conflict = Array.isArray(options.conflict) ? options.conflict : [];
|
|
94
|
+
return {
|
|
95
|
+
/**
|
|
96
|
+
* Writes records to PostgreSQL table.
|
|
97
|
+
*
|
|
98
|
+
* @param {Array} records - Array of objects with keys matching columns
|
|
99
|
+
* @param {object} [client] - Optional pg client/pool with query()
|
|
100
|
+
* @returns {Promise} Promise resolving when insert completes
|
|
101
|
+
*/
|
|
102
|
+
write(records, client) {
|
|
103
|
+
if (stopping) { return Promise.reject(new Error('PostgreSQL sink stopped')); }
|
|
104
|
+
if (client?.signal?.aborted) { return Promise.reject(client.signal.reason || new Error('PostgreSQL write aborted')); }
|
|
105
|
+
const unique = deduplicate(records, conflict);
|
|
106
|
+
const placeholders = unique.map((record, i) => {
|
|
107
|
+
const offset = i * columns.length;
|
|
108
|
+
const row = columns.map((__, j) => {return `$${offset + j + 1}`}).join(', ');
|
|
109
|
+
return `(${row})`;
|
|
110
|
+
}).join(', ');
|
|
111
|
+
const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
|
|
112
|
+
const values = unique.flatMap((record) => {return columns.map((col) => {return record[col]})});
|
|
113
|
+
const runner = client && typeof client.query === 'function' ? client : pool;
|
|
114
|
+
return runner.query(query, values);
|
|
115
|
+
},
|
|
116
|
+
stop() {
|
|
117
|
+
stopping ??= deadline(Promise.resolve().then(() => {
|
|
118
|
+
if (!options.pool) { return pool.end(); }
|
|
119
|
+
return undefined;
|
|
120
|
+
}), options.stopTimeoutMs ?? 5000, 'PostgreSQL pool close');
|
|
121
|
+
return stopping;
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import deadline from './deadline.js';
|
|
2
|
+
|
|
3
|
+
export function report(options, error, device) {
|
|
4
|
+
try { options.onError?.(error, device); } catch {
|
|
5
|
+
// Diagnostics cannot prevent shutdown.
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
export function closePort(port, timeoutMs) {
|
|
9
|
+
if (port.isOpen === false) { return Promise.resolve(); }
|
|
10
|
+
return deadline(new Promise((resolve, reject) => {
|
|
11
|
+
port.close((error) => { if (error) { reject(error); } else { resolve(); } });
|
|
12
|
+
}), timeoutMs, 'Serial close', () => { port.destroy?.(); }).catch((error) => {
|
|
13
|
+
error.resourceUncertain = true;
|
|
14
|
+
throw error;
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
function reserve(ctx, size) {
|
|
18
|
+
try {
|
|
19
|
+
const release = ctx.options.budget?.reserve({ bytes: size, records: 1 });
|
|
20
|
+
ctx.port.pause?.();
|
|
21
|
+
return { release };
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
Object.assign(ctx.state, { fatal: error, active: false });
|
|
25
|
+
ctx.state.controller.abort(error);
|
|
26
|
+
ctx.fail(error);
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function accept(ctx, chunk, release) {
|
|
31
|
+
const accepted = Promise.resolve().then(() => {
|
|
32
|
+
if (ctx.state.active && !ctx.failed) { return ctx.collector.accept({ device: ctx.device, chunk }, { signal: ctx.state.controller.signal }); }
|
|
33
|
+
return undefined;
|
|
34
|
+
});
|
|
35
|
+
void accepted.finally(() => { release?.(); }).catch(() => { return undefined; });
|
|
36
|
+
return deadline(accepted, ctx.options.acceptTimeoutMs, 'Serial acceptance', (error) => {
|
|
37
|
+
ctx.state.fatal = error; ctx.state.active = false; ctx.state.controller.abort(error);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
function enqueue(ctx, chunk) {
|
|
41
|
+
if (!ctx.state.active) { return; }
|
|
42
|
+
const size = chunk.length ?? Buffer.byteLength(String(chunk));
|
|
43
|
+
if (ctx.count >= ctx.options.maxPending || ctx.bytes + size > ctx.options.maxPendingBytes) {
|
|
44
|
+
const error = new Error('Serial receive capacity exceeded; source retired');
|
|
45
|
+
Object.assign(ctx.state, { fatal: error, active: false });
|
|
46
|
+
ctx.state.controller.abort(error);
|
|
47
|
+
ctx.fail(error);
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const lease = reserve(ctx, size);
|
|
51
|
+
if (!lease) { return; }
|
|
52
|
+
Object.assign(ctx, { count: ctx.count + 1, bytes: ctx.bytes + size });
|
|
53
|
+
ctx.pending = ctx.pending.then(() => {
|
|
54
|
+
if (ctx.state.active && !ctx.failed) {
|
|
55
|
+
return accept(ctx, chunk, lease.release);
|
|
56
|
+
}
|
|
57
|
+
lease.release?.();
|
|
58
|
+
return undefined;
|
|
59
|
+
}).catch(ctx.fail).finally(() => {
|
|
60
|
+
ctx.count -= 1;
|
|
61
|
+
ctx.bytes -= size;
|
|
62
|
+
if (!ctx.count && ctx.state.active && !ctx.failed) { ctx.port.resume?.(); }
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
/** Bound data even if a custom driver ignores pause; pause/resume the native stream. */
|
|
66
|
+
export default async function receive(state, port, device, collector, options) {
|
|
67
|
+
const ctx = { state, port, device, collector, options, pending: Promise.resolve(), count: 0, bytes: 0, failed: false };
|
|
68
|
+
const closed = new Promise((resolve) => { state.wake = resolve; });
|
|
69
|
+
ctx.fail = (error) => { ctx.failed = true; report(options, error, device); state.wake?.(); };
|
|
70
|
+
function data(chunk) { enqueue(ctx, chunk); }
|
|
71
|
+
function close() { state.wake?.(); }
|
|
72
|
+
port.on('data', data).on('error', ctx.fail).on('close', close);
|
|
73
|
+
if (!state.active) { state.wake(); }
|
|
74
|
+
await closed;
|
|
75
|
+
port.off('data', data);
|
|
76
|
+
try {
|
|
77
|
+
await closePort(port, options.stopTimeoutMs);
|
|
78
|
+
await deadline(ctx.pending, options.stopTimeoutMs, 'Serial acceptance', (error) => { state.controller.abort(error); });
|
|
79
|
+
} finally { port.off('error', ctx.fail).off('close', close); }
|
|
80
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import positive from './positive.js';
|
|
2
|
+
|
|
3
|
+
import deadline from './deadline.js';
|
|
4
|
+
import receive, { closePort, report } from './serialReceive.js';
|
|
5
|
+
|
|
6
|
+
async function openPort(device) {
|
|
7
|
+
const { SerialPort } = await import('serialport');
|
|
8
|
+
const port = new SerialPort({ ...device, autoOpen: false });
|
|
9
|
+
await new Promise((resolve, reject) => { port.open((error) => { if (error) { reject(error); } else { resolve(); } }); });
|
|
10
|
+
return port;
|
|
11
|
+
}
|
|
12
|
+
function pause(state, milliseconds) {
|
|
13
|
+
return new Promise((resolve) => { state.wake = resolve; state.timer = setTimeout(resolve, milliseconds); })
|
|
14
|
+
.finally(() => { clearTimeout(state.timer); state.wake = null; });
|
|
15
|
+
}
|
|
16
|
+
async function run(state, device, collector, options) {
|
|
17
|
+
while (state.active) {
|
|
18
|
+
try {
|
|
19
|
+
// Each port has an independent serialized open/close/retry loop.
|
|
20
|
+
const attempt = { timedOut: false };
|
|
21
|
+
const opening = Promise.resolve().then(() => { return options.openPort(device, { signal: state.controller.signal }); });
|
|
22
|
+
void opening.then((late) => {
|
|
23
|
+
if (attempt.timedOut) { return closePort(late, options.stopTimeoutMs); }
|
|
24
|
+
return undefined;
|
|
25
|
+
}).catch((error) => { report(options, error, device); });
|
|
26
|
+
// A timed-out open is uncertain; never open a second handle in this instance.
|
|
27
|
+
// eslint-disable-next-line no-await-in-loop
|
|
28
|
+
const port = await deadline(opening, options.openTimeoutMs, 'Serial open', (error) => { attempt.timedOut = true; state.controller.abort(error); });
|
|
29
|
+
// eslint-disable-next-line no-await-in-loop
|
|
30
|
+
await receive(state, port, device, collector, options);
|
|
31
|
+
} catch (error) {
|
|
32
|
+
if (error.resourceUncertain || error.code === 'OPERATION_TIMEOUT') { Object.assign(state, { fatal: error, active: false }); }
|
|
33
|
+
report(options, error, device);
|
|
34
|
+
}
|
|
35
|
+
if (state.active) {
|
|
36
|
+
// eslint-disable-next-line no-await-in-loop
|
|
37
|
+
await pause(state, options.retryMs);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function validate(devices, collector) {
|
|
42
|
+
if (!Array.isArray(devices)) { throw new TypeError('devices must be an array'); }
|
|
43
|
+
if (!collector || typeof collector.accept !== 'function') { throw new TypeError('Collector must have an accept() method'); }
|
|
44
|
+
const paths = new Set();
|
|
45
|
+
devices.forEach((device) => {
|
|
46
|
+
if (!device || typeof device.path !== 'string' || !device.path.trim() || paths.has(device.path)) {
|
|
47
|
+
throw new TypeError('invalid or duplicate serial path');
|
|
48
|
+
}
|
|
49
|
+
paths.add(device.path);
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
/** Independent serial devices; chunks and device metadata pass through unchanged. */
|
|
53
|
+
export default function serialSource(devices, collector, options = {}) {
|
|
54
|
+
validate(devices, collector);
|
|
55
|
+
const settings = { retryMs: 1000, openTimeoutMs: 5000, acceptTimeoutMs: 10000, stopTimeoutMs: 5000, maxPending: 32, maxPendingBytes: 1048576,
|
|
56
|
+
...options, openPort: options.openPort ?? openPort };
|
|
57
|
+
['retryMs', 'acceptTimeoutMs', 'openTimeoutMs', 'stopTimeoutMs', 'maxPending', 'maxPendingBytes'].forEach((key) => { positive(settings[key], key); });
|
|
58
|
+
if (!Number.isSafeInteger(settings.maxPending) || !Number.isSafeInteger(settings.maxPendingBytes)) { throw new TypeError('Serial capacities must be safe integers'); }
|
|
59
|
+
const states = devices.map(() => { return { active: false, task: null, wake: null, timer: null }; });
|
|
60
|
+
let revision = 0;
|
|
61
|
+
let stopping = Promise.resolve();
|
|
62
|
+
return {
|
|
63
|
+
async start() {
|
|
64
|
+
revision += 1;
|
|
65
|
+
const requested = revision;
|
|
66
|
+
await stopping;
|
|
67
|
+
const failed = states.find((state) => { return state.fatal; });
|
|
68
|
+
if (failed) { throw failed.fatal; }
|
|
69
|
+
if (requested === revision) {
|
|
70
|
+
states.forEach((state, index) => {
|
|
71
|
+
if (!state.active) { state.active = true; state.controller = new AbortController(); state.task = run(state, devices[index], collector, settings); }
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
stop() {
|
|
76
|
+
revision += 1;
|
|
77
|
+
states.forEach((state) => { state.active = false; state.controller?.abort(new Error('Serial source stopped')); state.wake?.(); });
|
|
78
|
+
stopping = deadline(Promise.all(states.map((state) => { return state.task; })), settings.stopTimeoutMs, 'Serial stop', (error) => {
|
|
79
|
+
states.forEach((state) => { state.fatal = error; });
|
|
80
|
+
}).then(() => {
|
|
81
|
+
const failed = states.find((state) => { return state.fatal; });
|
|
82
|
+
if (failed) { throw failed.fatal; }
|
|
83
|
+
});
|
|
84
|
+
return stopping;
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
}
|