@yarkivaev/scada 2.3.115 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/README.md +158 -0
  2. package/charts/scada/templates/processes.yaml +22 -0
  3. package/charts/scada/values.schema.json +12 -0
  4. package/db/migrations/V0009__collector_task_state.sql +6 -0
  5. package/package.json +14 -5
  6. package/src/application/deploy/collectorProcesses.js +3 -2
  7. package/src/application/deploy/readDocument.js +4 -1
  8. package/src/application/plantServer.js +1 -1
  9. package/src/application/runtime/additionalAdapters.js +139 -0
  10. package/src/application/runtime/assignment.js +30 -0
  11. package/src/application/runtime/budgetedSink.js +17 -0
  12. package/src/application/runtime/collectorSignals.js +14 -0
  13. package/src/application/runtime/compositeLifecycle.js +49 -0
  14. package/src/application/runtime/createRegistry.js +20 -0
  15. package/src/application/runtime/deadline.js +8 -0
  16. package/src/application/runtime/index.js +10 -0
  17. package/src/application/runtime/manifest.js +38 -0
  18. package/src/application/runtime/pipelineAdapters.js +60 -0
  19. package/src/application/runtime/resolveSecrets.js +12 -0
  20. package/src/application/runtime/resourceClaims.js +21 -0
  21. package/src/application/runtime/runCollector.js +50 -0
  22. package/src/application/runtime/runtime.js +86 -0
  23. package/src/application/runtime/runtimeHealth.js +29 -0
  24. package/src/application/runtime/runtimeServices.js +15 -0
  25. package/src/application/runtime/stateDocument.js +22 -0
  26. package/src/application/runtime/streamPorts.js +42 -0
  27. package/src/application/runtime/streamTask.js +105 -0
  28. package/src/application/runtime/taskBudget.js +31 -0
  29. package/src/application/runtime/taskContext.js +18 -0
  30. package/src/application/runtime/taskQueue.js +21 -0
  31. package/src/application/runtime/taskSlot.js +46 -0
  32. package/src/application/runtime/taskState.js +73 -0
  33. package/src/application/siteCollection.js +14 -0
  34. package/src/application/siteServer.js +7 -6
  35. package/src/application/siteTelemetry.js +1 -1
  36. package/src/application/supervisorSink.js +9 -8
  37. package/src/application/sync/siteSyncBind.js +1 -1
  38. package/src/application/sync/siteSyncTargets.js +1 -1
  39. package/src/bin/collector.js +7 -0
  40. package/src/bin/modbus-to-mqtt.js +11 -9
  41. package/src/infrastructure/deploy/executeRelease.js +45 -3
  42. package/src/infrastructure/ingest/activity/activityTracking.js +15 -21
  43. package/src/infrastructure/ingest/activity/activityTransformer.js +16 -15
  44. package/src/infrastructure/ingest/durableConsume.js +2 -169
  45. package/src/infrastructure/ingest/ingestBatch.js +13 -0
  46. package/src/infrastructure/ingest/modbus/mx210Tcp.js +4 -4
  47. package/src/infrastructure/ingest/modbus/silentStreams.js +39 -16
  48. package/src/infrastructure/ingest/mqtt/metricsTransformer.js +4 -5
  49. package/src/infrastructure/ingest/mqtt/modbusMqtt.js +17 -19
  50. package/src/infrastructure/ingest/mqtt/modbusPublish.js +4 -0
  51. package/src/infrastructure/ingest/mqtt/mqttMetrics.js +17 -21
  52. package/src/infrastructure/ingest/mqtt/registerPublish.js +3 -3
  53. package/src/infrastructure/ingest/pipelines/alertPipeline.js +7 -6
  54. package/src/infrastructure/ingest/pipelines/decisionPipeline.js +7 -6
  55. package/src/infrastructure/ingest/pipelines/segmentPipeline.js +8 -17
  56. package/src/infrastructure/ingest/pipelines/stompOptions.js +7 -0
  57. package/src/infrastructure/ingest/silentOpenWatch.js +5 -42
  58. package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js +16 -28
  59. package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js +17 -22
  60. package/src/infrastructure/ingest/telemetry/foldingSink.js +5 -2
  61. package/src/infrastructure/messaging/stomp/segmentRetags.js +1 -1
  62. package/src/infrastructure/messaging/stomp/userDecisions.js +1 -1
  63. package/src/infrastructure/opcua/certificates.js +11 -0
  64. package/src/infrastructure/opcua/config.js +38 -0
  65. package/src/infrastructure/opcua/delivery.js +38 -0
  66. package/src/infrastructure/opcua/endpoint.js +90 -0
  67. package/src/infrastructure/opcua/index.js +10 -0
  68. package/src/infrastructure/opcua/session.js +70 -0
  69. package/src/infrastructure/opcua/source.js +38 -0
  70. package/src/infrastructure/opcua/subscriptionHealth.js +28 -0
  71. package/src/infrastructure/operators/operatorsSync.js +5 -29
  72. package/src/infrastructure/persistence/clickhouse/streamHub.js +21 -15
  73. package/src/infrastructure/persistence/metricsSensor.js +9 -6
  74. package/src/infrastructure/persistence/runtime/taskStatePg.js +24 -0
  75. package/src/infrastructure/streams/README.md +291 -0
  76. package/src/infrastructure/streams/abortable.js +14 -0
  77. package/src/infrastructure/streams/amqpSink/publication.js +46 -0
  78. package/src/infrastructure/streams/amqpSink/session.js +96 -0
  79. package/src/infrastructure/streams/amqpSink.js +47 -0
  80. package/src/infrastructure/streams/amqpSource/session.js +127 -0
  81. package/src/infrastructure/streams/amqpSource.js +97 -0
  82. package/src/infrastructure/streams/brokerClose.js +19 -0
  83. package/src/infrastructure/streams/budgetAccept.js +6 -0
  84. package/src/infrastructure/streams/clickhouseSink.js +48 -0
  85. package/src/infrastructure/streams/deadline.js +1 -0
  86. package/src/infrastructure/streams/follow.js +41 -0
  87. package/src/infrastructure/streams/index.js +15 -0
  88. package/src/infrastructure/streams/lokiSource.js +88 -0
  89. package/src/infrastructure/streams/modbusConnection.js +173 -0
  90. package/src/infrastructure/streams/modbusRtuBusSource.js +124 -0
  91. package/src/infrastructure/streams/modbusSource.js +185 -0
  92. package/src/infrastructure/streams/mqttSink/session.js +51 -0
  93. package/src/infrastructure/streams/mqttSink.js +54 -0
  94. package/src/infrastructure/streams/mqttSource.js +66 -0
  95. package/src/infrastructure/streams/pollingSource.js +1 -0
  96. package/src/infrastructure/streams/positive.js +5 -0
  97. package/src/infrastructure/streams/postgresSink.js +124 -0
  98. package/src/infrastructure/streams/serialReceive.js +80 -0
  99. package/src/infrastructure/streams/serialSource.js +87 -0
  100. package/src/infrastructure/streams/sqliteSink.js +55 -0
  101. package/src/infrastructure/streams/stompBody.js +46 -0
  102. package/src/infrastructure/streams/stompSend.js +89 -0
  103. package/src/infrastructure/streams/stompSource.js +201 -0
  104. package/src/infrastructure/sync/operationSyncIngest.js +4 -16
@@ -0,0 +1,185 @@
1
+ import pollingSource from './pollingSource.js';
2
+ import deadline from './deadline.js';
3
+ import abortable from './abortable.js';
4
+ import { attempt, create, drop, idle, opening, resume } from './modbusConnection.js';
5
+
6
+ function read(bag, address, count) {
7
+ return async function fetch(since, until, { signal } = {}) {
8
+ const { client } = bag;
9
+ try {
10
+ const result = await abortable(client.readHoldingRegisters(address, count), signal, () => { drop(client); });
11
+ return [result.data];
12
+ } catch (err) {
13
+ if (bag.client === client) { bag.lost(); }
14
+ throw err;
15
+ }
16
+ };
17
+ }
18
+
19
+ /**
20
+ * Shared Modbus polling lifecycle for TCP and RTU transports.
21
+ *
22
+ * @param {object} options - connect, target, address, count, interval, collector, clk, client, log
23
+ * @returns {object} Source with start() and stop() methods
24
+ */
25
+ function modbusPollingSource(options) {
26
+ const { address, count, interval, collector, clk } = options;
27
+ const bag = {
28
+ ...options,
29
+ client: undefined,
30
+ state: idle(),
31
+ epoch: 0,
32
+ maxDelay: 300,
33
+ delay: interval
34
+ };
35
+ bag.lost = function lost() {
36
+ resume(bag);
37
+ };
38
+ bag.source = pollingSource(read(bag, address, count), interval, collector, clk);
39
+ return {
40
+ /**
41
+ * Connects to the Modbus device and starts polling.
42
+ */
43
+ start() {
44
+ if (bag.fatal) { return Promise.reject(bag.fatal); }
45
+ if (bag.state.connected() || bag.state.retrying() || bag.state.opening()) {
46
+ return Promise.resolve();
47
+ }
48
+ const token = bag.epoch;
49
+ bag.state = opening();
50
+ return Promise.resolve(bag.stopping).then(() => { if (token === bag.epoch) { attempt(bag); } });
51
+ },
52
+ /**
53
+ * Stops polling and disconnects from the Modbus device.
54
+ */
55
+ stop() {
56
+ bag.epoch += 1;
57
+ const drained = bag.source.stop();
58
+ if (bag.state.retrying()) {
59
+ bag.state.cancel();
60
+ }
61
+ const { client } = bag;
62
+ bag.client = undefined;
63
+ bag.state = idle();
64
+ drop(client);
65
+ bag.stopping = deadline(Promise.all([drained, bag.opened]), 5000, 'Modbus stop', (error) => { bag.fatal = error; }).then(() => {
66
+ if (bag.fatal) { throw bag.fatal; }
67
+ });
68
+ return bag.stopping;
69
+ }
70
+ };
71
+ }
72
+
73
+ /**
74
+ * Modbus TCP polling source for reading holding registers.
75
+ *
76
+ * Connects to a Modbus TCP device and polls holding registers
77
+ * at a specified interval. Forwards register arrays to the
78
+ * collector via pollingSource.
79
+ *
80
+ * @example
81
+ * const source = modbusSource('192.168.2.148', 502, 4000, 14, 5, collector, clock);
82
+ * source.start();
83
+ *
84
+ * @param {string} host - Modbus TCP host address
85
+ * @param {number} port - Modbus TCP port
86
+ * @param {number} address - Starting register address
87
+ * @param {number} count - Number of registers to read
88
+ * @param {number} interval - Polling interval in seconds
89
+ * @param {object} collector - Collector with accept() method
90
+ * @param {object} clk - Clock with millis() method
91
+ * @param {object} [log] - Logger with error() method for connection failures
92
+ * @returns {object} Source with start() and stop() methods
93
+ */
94
+ // Preserve the established positional adapter API.
95
+ // eslint-disable-next-line max-params
96
+ export default function modbusSource(host, port, address, count, interval, collector, clk, log = console) {
97
+ if (typeof host !== 'string' || host.length === 0) {
98
+ throw new Error('Host must be a non-empty string');
99
+ }
100
+ if (typeof port !== 'number' || port <= 0) {
101
+ throw new Error(`Port must be a positive number, got: ${port}`);
102
+ }
103
+ if (typeof address !== 'number' || address < 0) {
104
+ throw new Error(`Address must be a non-negative number, got: ${address}`);
105
+ }
106
+ if (typeof count !== 'number' || count <= 0) {
107
+ throw new Error(`Count must be a positive number, got: ${count}`);
108
+ }
109
+ function connect(modbusClient) {
110
+ return modbusClient.connectTCP(host, { port });
111
+ }
112
+ return modbusPollingSource({
113
+ create,
114
+ connect,
115
+ target: `${host}:${port}`,
116
+ address,
117
+ count,
118
+ interval,
119
+ collector,
120
+ clk,
121
+ log
122
+ });
123
+ }
124
+
125
+ /**
126
+ * Modbus RTU polling source for reading holding registers over serial.
127
+ *
128
+ * Connects to a Modbus RTU device on a serial port and polls holding registers
129
+ * at a specified interval. Forwards register arrays to the collector via pollingSource.
130
+ *
131
+ * @example
132
+ * const source = modbusRtuSource('/dev/ttyUSB0', { baudRate: 9600, slaveId: 1 }, 4000, 44, 5, collector, clock);
133
+ * source.start();
134
+ *
135
+ * @param {string} path - Serial device path (e.g. /dev/ttyUSB0)
136
+ * @param {object} serial - baudRate, dataBits, stopBits, parity, slaveId
137
+ * @param {number} address - Starting register address
138
+ * @param {number} count - Number of registers to read
139
+ * @param {number} interval - Polling interval in seconds
140
+ * @param {object} collector - Collector with accept() method
141
+ * @param {object} clk - Clock with millis() method
142
+ * @param {object} [log] - Logger with error() method for connection failures
143
+ * @returns {object} Source with start() and stop() methods
144
+ */
145
+ // Preserve the established positional adapter API.
146
+ // eslint-disable-next-line max-params
147
+ export function modbusRtuSource(path, serial, address, count, interval, collector, clk, log = console) {
148
+ if (typeof path !== 'string' || path.length === 0) {
149
+ throw new Error('Serial path must be a non-empty string');
150
+ }
151
+ if (!serial || typeof serial !== 'object') {
152
+ throw new Error('Serial config must be an object');
153
+ }
154
+ if (typeof serial.baudRate !== 'number' || serial.baudRate <= 0) {
155
+ throw new Error(`Baud rate must be a positive number, got: ${serial.baudRate}`);
156
+ }
157
+ if (typeof address !== 'number' || address < 0) {
158
+ throw new Error(`Address must be a non-negative number, got: ${address}`);
159
+ }
160
+ if (typeof count !== 'number' || count <= 0) {
161
+ throw new Error(`Count must be a positive number, got: ${count}`);
162
+ }
163
+ const slaveId = serial.slaveId || 1;
164
+ function connect(modbusClient) {
165
+ return modbusClient.connectRTUBuffered(path, {
166
+ baudRate: serial.baudRate,
167
+ dataBits: serial.dataBits || 8,
168
+ stopBits: serial.stopBits || 1,
169
+ parity: serial.parity || 'none'
170
+ }).then(() => {
171
+ modbusClient.setID(slaveId);
172
+ });
173
+ }
174
+ return modbusPollingSource({
175
+ create,
176
+ connect,
177
+ target: path,
178
+ address,
179
+ count,
180
+ interval,
181
+ collector,
182
+ clk,
183
+ log
184
+ });
185
+ }
@@ -0,0 +1,51 @@
1
+ import deadline from '../deadline.js';
2
+
3
+ export function terminate(live, error) {
4
+ if (live.dead) { return live.closing; }
5
+ live.dead = true;
6
+ live.ready = false;
7
+ live.pending.forEach((settle) => { settle(error || new Error('MQTT connection closed')); });
8
+ live.closing = deadline(new Promise((resolve) => {
9
+ live.client.end(true, {}, resolve);
10
+ }), live.options.stopTimeoutMs, 'MQTT disconnect');
11
+ // An event callback cannot surface a rejected shutdown. stop() still returns it.
12
+ void live.closing.catch(() => { return undefined; });
13
+ return live.closing;
14
+ }
15
+ export function connect(live) {
16
+ return deadline(new Promise((resolve, reject) => {
17
+ function fail(error) { reject(error); void terminate(live, error); }
18
+ live.client.on('error', fail);
19
+ live.client.on('close', () => { fail(new Error('MQTT connection closed')); });
20
+ live.client.on('offline', () => { fail(new Error('MQTT connection offline')); });
21
+ live.client.on('connect', () => {
22
+ if (!live.dead) { live.ready = true; resolve(); }
23
+ });
24
+ live.pending.add(reject);
25
+ }), live.options.connectTimeoutMs, 'MQTT connect', (error) => { void terminate(live, error); });
26
+ }
27
+ export function publish(live, record, signal) {
28
+ return new Promise((resolve, reject) => {
29
+ let done = false;
30
+ const timer = { handle: null };
31
+ function abort() { void terminate(live, signal.reason || new Error('MQTT publish aborted')); }
32
+ function settle(error) {
33
+ if (done) { return; }
34
+ done = true;
35
+ clearTimeout(timer.handle);
36
+ signal?.removeEventListener('abort', abort);
37
+ live.pending.delete(settle);
38
+ live.count -= 1;
39
+ if (error) { reject(error); } else { resolve(); }
40
+ }
41
+ timer.handle = setTimeout(() => { void terminate(live, new Error('MQTT confirmation timed out; outcome uncertain')); }, live.options.confirmTimeoutMs);
42
+ live.pending.add(settle);
43
+ signal?.addEventListener('abort', abort, { once: true });
44
+ if (signal?.aborted) { abort(); return; }
45
+ try { live.client.publish(record.topic, record.payload, { qos: live.options.qos }, (error) => {
46
+ settle(error);
47
+ if (error) { void terminate(live, error); }
48
+ }); }
49
+ catch (error) { void terminate(live, error); }
50
+ });
51
+ }
@@ -0,0 +1,54 @@
1
+ import mqtt from 'mqtt';
2
+ import positive from './positive.js';
3
+ import { connect, publish, terminate } from './mqttSink/session.js';
4
+
5
+ function settings(options) {
6
+ const config = { qos: 0, maxPending: 1024, maxPendingBytes: 1048576, connectTimeoutMs: 5000, confirmTimeoutMs: 10000, stopTimeoutMs: 5000, ...options };
7
+ ['maxPending', 'maxPendingBytes', 'connectTimeoutMs', 'confirmTimeoutMs', 'stopTimeoutMs'].forEach((key) => { positive(config[key], key); });
8
+ if (!Number.isSafeInteger(config.maxPending)) { throw new TypeError('maxPending must be a safe integer'); }
9
+ if (![0, 1, 2].includes(config.qos)) { throw new TypeError('qos must be 0, 1, or 2'); }
10
+ return config;
11
+ }
12
+ function write(live, records, signal) {
13
+ if (!live?.ready || live.dead) { throw new Error('Sink is not connected'); }
14
+ if (!Array.isArray(records) || records.some((record) => {
15
+ return !record || typeof record.topic !== 'string' || !record.topic || !(typeof record.payload === 'string' || Buffer.isBuffer(record.payload));
16
+ })) { return Promise.reject(new TypeError('MQTT records require a topic and string/Buffer payload')); }
17
+ if (signal?.aborted) { return Promise.reject(signal.reason || new Error('MQTT publish aborted')); }
18
+ const bytes = records.reduce((total, record) => { return total + Buffer.byteLength(record.payload) + Buffer.byteLength(record.topic); }, 0);
19
+ if (live.bytes + bytes > live.options.maxPendingBytes || live.count + records.length > live.options.maxPending) { return Promise.reject(new Error('MQTT pending capacity exceeded')); }
20
+ live.count += records.length;
21
+ live.bytes += bytes;
22
+ return Promise.all(records.map((record) => {
23
+ if (live.dead) { live.count -= 1; return Promise.reject(new Error('MQTT connection closed')); }
24
+ return publish(live, record, signal);
25
+ })).finally(() => { live.bytes -= bytes; });
26
+ }
27
+ /** start awaits CONNACK; write awaits publish completion (broker confirmation at QoS 1/2).
28
+ * QoS 0 only confirms local handoff. No automatic replay of uncertain writes.
29
+ */
30
+ export default function mqttSink(url, options = {}) {
31
+ if (typeof url !== 'string' || !url) { throw new Error('URL must be a non-empty string'); }
32
+ const config = settings(options);
33
+ let live;
34
+ return {
35
+ start() {
36
+ if (live && !live.dead) { return live.opening; }
37
+ if (live?.dead) { return Promise.reject(new Error('MQTT session retired; create a new sink after stop')); }
38
+ const client = (options.connect || mqtt.connect)(url, {
39
+ ...options.connectionOptions, clientId: options.clientId, protocolVersion: 5,
40
+ reconnectPeriod: 0, queueQoSZero: false, connectTimeout: config.connectTimeoutMs
41
+ });
42
+ live = { client, options: config, pending: new Set(), count: 0, bytes: 0, ready: false, dead: false };
43
+ live.opening = connect(live);
44
+ return live.opening;
45
+ },
46
+ write(records, { signal } = {}) { return write(live, records, signal); },
47
+ stop() {
48
+ const stopped = live;
49
+ return (stopped ? terminate(stopped, new Error('MQTT sink stopped')) : Promise.resolve()).then(() => {
50
+ if (live === stopped) { live = undefined; }
51
+ });
52
+ }
53
+ };
54
+ }
@@ -0,0 +1,66 @@
1
+ import mqtt from 'mqtt';
2
+ import positive from './positive.js';
3
+ import deadline from './deadline.js';
4
+ import budgetAccept from './budgetAccept.js';
5
+
6
+ function report(options, error) {
7
+ try { options.onError?.(error); } catch {
8
+ // Diagnostics cannot break transport callbacks.
9
+ }
10
+ }
11
+ function bind(live, collector, options) {
12
+ live.client.on('error', (error) => { report(options, error); });
13
+ // MQTT.js serializes handleMessage and sends PUBACK/PUBCOMP only on successful completion.
14
+ live.client.handleMessage = (packet, callback) => {
15
+ const accepted = Promise.resolve().then(() => {
16
+ if (!live.active) { throw new Error('MQTT source stopped'); }
17
+ return budgetAccept(collector, { topic: packet.topic, payload: packet.payload.toString() }, packet.payload.length, options, { signal: live.controller.signal });
18
+ });
19
+ live.pending.add(accepted);
20
+ void accepted.finally(() => { live.pending.delete(accepted); }).catch(() => { return undefined; });
21
+ deadline(accepted, options.acceptTimeoutMs, 'MQTT acceptance', (error) => {
22
+ live.active = false; live.controller.abort(error); live.client.end(true);
23
+ }).then(() => {
24
+ callback(live.active ? undefined : new Error('MQTT source stopped'));
25
+ }, (error) => { callback(error); report(options, error); });
26
+ };
27
+ }
28
+ function open(url, list, collector, options) {
29
+ const client = (options.connect || mqtt.connect)(url, {
30
+ ...options.connectionOptions, clientId: options.clientId, clean: !options.clientId, protocolVersion: 5,
31
+ properties: { sessionExpiryInterval: options.clientId ? options.sessionExpiryInterval ?? 3600 : 0,
32
+ receiveMaximum: options.prefetch ?? 32, maximumPacketSize: options.maxPacketBytes ?? 1048576 }
33
+ });
34
+ const live = { client, active: true, pending: new Set(), controller: new AbortController() };
35
+ bind(live, collector, options);
36
+ client.on('connect', () => {
37
+ if (live.active) { client.subscribe(list, { qos: options.qos ?? (options.clientId ? 1 : 0) }); }
38
+ });
39
+ return live;
40
+ }
41
+ /** ACK follows asynchronous acceptance, not merely receipt. Acceptance is not necessarily durable. */
42
+ export default function mqttSource(url, topics, collector, options = {}) {
43
+ if (typeof url !== 'string' || !url) { throw new Error('URL must be a non-empty string'); }
44
+ if (typeof topics !== 'string' || !topics.trim()) { throw new Error('Topic must be a non-empty string'); }
45
+ if (typeof collector?.accept !== 'function') { throw new Error('Collector must have an accept() method'); }
46
+ const config = { acceptTimeoutMs: 10000, stopTimeoutMs: 5000, ...options };
47
+ positive(config.acceptTimeoutMs, 'acceptTimeoutMs');
48
+ positive(config.stopTimeoutMs, 'stopTimeoutMs');
49
+ const list = topics.split(',').map((topic) => { return topic.trim(); }).filter(Boolean);
50
+ let live;
51
+ return {
52
+ start() {
53
+ if (!live) { live = open(url, list, collector, config); }
54
+ },
55
+ stop() {
56
+ if (!live) { return Promise.resolve(); }
57
+ const stopped = live;
58
+ stopped.active = false;
59
+ stopped.controller.abort(new Error('MQTT source stopped'));
60
+ const closing = new Promise((resolve) => { stopped.client.end(true, {}, resolve); });
61
+ return deadline(Promise.all([closing, ...stopped.pending]), config.stopTimeoutMs, 'MQTT stop').then(() => {
62
+ if (live === stopped) { live = undefined; }
63
+ });
64
+ }
65
+ };
66
+ }
@@ -0,0 +1 @@
1
+ export { pollingSource as default } from '@yarkivaev/source-to-sink';
@@ -0,0 +1,5 @@
1
+ export default function positive(value, name) {
2
+ if (!Number.isFinite(value) || value <= 0) {
3
+ throw new TypeError(`${name} must be a positive number`);
4
+ }
5
+ }
@@ -0,0 +1,124 @@
1
+ import pg from 'pg';
2
+ import deadline from './deadline.js';
3
+ import positive from './positive.js';
4
+
5
+ /**
6
+ * Removes duplicate records by conflict key within a batch.
7
+ *
8
+ * PostgreSQL rejects INSERT batches where ON CONFLICT would update
9
+ * the same row twice. This keeps the last occurrence for each
10
+ * unique combination of conflict column values.
11
+ *
12
+ * @example
13
+ * deduplicate(
14
+ * [{ machine: 'a', start: 1, name: 'pending' },
15
+ * { machine: 'a', start: 1, name: 'completed' }],
16
+ * ['machine', 'start']
17
+ * );
18
+ * // => [{ machine: 'a', start: 1, name: 'completed' }]
19
+ *
20
+ * @param {Array} records - Array of record objects
21
+ * @param {Array<string>} keys - Conflict column names
22
+ * @returns {Array} Deduplicated records
23
+ */
24
+ export function deduplicate(records, keys) {
25
+ if (keys.length === 0) {return records;}
26
+ const seen = new Map();
27
+ for (const record of records) {
28
+ const key = keys.map(k => {return record[k]}).join('\0');
29
+ seen.set(key, record);
30
+ }
31
+ return Array.from(seen.values());
32
+ }
33
+
34
+ /**
35
+ * Builds the ON CONFLICT SQL suffix from options.
36
+ *
37
+ * @param {object} options - Sink options with conflict, update, and optional updateWhere
38
+ * @returns {string} SQL suffix or empty string
39
+ */
40
+ function buildSuffix(options) {
41
+ if (!Array.isArray(options.conflict) || options.conflict.length === 0) {return '';}
42
+ const cols = options.conflict.join(', ');
43
+ if (Array.isArray(options.update) && options.update.length > 0) {
44
+ const sets = options.update.map((col) => {return `${col} = EXCLUDED.${col}`}).join(', ');
45
+ const guard = typeof options.updateWhere === 'string' && options.updateWhere.length > 0
46
+ ? ` WHERE ${options.updateWhere}`
47
+ : '';
48
+ return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}${guard}`;
49
+ }
50
+ return ` ON CONFLICT (${cols}) DO NOTHING`;
51
+ }
52
+
53
+ /**
54
+ * PostgreSQL sink for batch record insertion.
55
+ *
56
+ * Creates a pg.Pool internally unless `options.pool` is supplied.
57
+ * Supports optional conflict resolution via ON CONFLICT DO NOTHING or
58
+ * ON CONFLICT DO UPDATE SET. `write` accepts an optional queryable
59
+ * client so callers can run inserts inside an existing transaction.
60
+ *
61
+ * @example
62
+ * const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
63
+ * await sink.write([{ ts: Date.now(), value: 42 }]);
64
+ *
65
+ * @example
66
+ * const sink = postgresSink(url, 'segments', cols, { pool, conflict: ['machine', 'start_time'] });
67
+ * await sink.write([{ machine: 'a', name: 'on', start_time: '2024-01-01' }], client);
68
+ *
69
+ * @param {string} url - PostgreSQL URL (e.g., 'postgresql://localhost:5432/db')
70
+ * @param {string} table - Target table name
71
+ * @param {Array<string>} columns - Column names for insertion
72
+ * @param {object} [options] - Optional configuration
73
+ * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
74
+ * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
75
+ * @param {string} [options.updateWhere] - Optional WHERE guard on DO UPDATE
76
+ * @param {object} [options.pool] - Existing pg Pool to reuse
77
+ * @returns {object} Sink with write(records, client?) method
78
+ */
79
+ export default function postgresSink(url, table, columns, options = {}) {
80
+ if (typeof url !== 'string' || url.length === 0) {
81
+ throw new Error('URL must be a non-empty string');
82
+ }
83
+ if (typeof table !== 'string' || table.length === 0) {
84
+ throw new Error('Table must be a non-empty string');
85
+ }
86
+ if (!Array.isArray(columns) || columns.length === 0) {
87
+ throw new Error('Columns must be a non-empty array');
88
+ }
89
+ positive(options.stopTimeoutMs ?? 5000, 'stopTimeoutMs');
90
+ const pool = options.pool || (options.createPool || ((config) => { return new pg.Pool(config); }))({ connectionString: url });
91
+ let stopping;
92
+ const suffix = buildSuffix(options);
93
+ const conflict = Array.isArray(options.conflict) ? options.conflict : [];
94
+ return {
95
+ /**
96
+ * Writes records to PostgreSQL table.
97
+ *
98
+ * @param {Array} records - Array of objects with keys matching columns
99
+ * @param {object} [client] - Optional pg client/pool with query()
100
+ * @returns {Promise} Promise resolving when insert completes
101
+ */
102
+ write(records, client) {
103
+ if (stopping) { return Promise.reject(new Error('PostgreSQL sink stopped')); }
104
+ if (client?.signal?.aborted) { return Promise.reject(client.signal.reason || new Error('PostgreSQL write aborted')); }
105
+ const unique = deduplicate(records, conflict);
106
+ const placeholders = unique.map((record, i) => {
107
+ const offset = i * columns.length;
108
+ const row = columns.map((__, j) => {return `$${offset + j + 1}`}).join(', ');
109
+ return `(${row})`;
110
+ }).join(', ');
111
+ const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
112
+ const values = unique.flatMap((record) => {return columns.map((col) => {return record[col]})});
113
+ const runner = client && typeof client.query === 'function' ? client : pool;
114
+ return runner.query(query, values);
115
+ },
116
+ stop() {
117
+ stopping ??= deadline(Promise.resolve().then(() => {
118
+ if (!options.pool) { return pool.end(); }
119
+ return undefined;
120
+ }), options.stopTimeoutMs ?? 5000, 'PostgreSQL pool close');
121
+ return stopping;
122
+ }
123
+ };
124
+ }
@@ -0,0 +1,80 @@
1
+ import deadline from './deadline.js';
2
+
3
+ export function report(options, error, device) {
4
+ try { options.onError?.(error, device); } catch {
5
+ // Diagnostics cannot prevent shutdown.
6
+ }
7
+ }
8
+ export function closePort(port, timeoutMs) {
9
+ if (port.isOpen === false) { return Promise.resolve(); }
10
+ return deadline(new Promise((resolve, reject) => {
11
+ port.close((error) => { if (error) { reject(error); } else { resolve(); } });
12
+ }), timeoutMs, 'Serial close', () => { port.destroy?.(); }).catch((error) => {
13
+ error.resourceUncertain = true;
14
+ throw error;
15
+ });
16
+ }
17
+ function reserve(ctx, size) {
18
+ try {
19
+ const release = ctx.options.budget?.reserve({ bytes: size, records: 1 });
20
+ ctx.port.pause?.();
21
+ return { release };
22
+ }
23
+ catch (error) {
24
+ Object.assign(ctx.state, { fatal: error, active: false });
25
+ ctx.state.controller.abort(error);
26
+ ctx.fail(error);
27
+ return null;
28
+ }
29
+ }
30
+ function accept(ctx, chunk, release) {
31
+ const accepted = Promise.resolve().then(() => {
32
+ if (ctx.state.active && !ctx.failed) { return ctx.collector.accept({ device: ctx.device, chunk }, { signal: ctx.state.controller.signal }); }
33
+ return undefined;
34
+ });
35
+ void accepted.finally(() => { release?.(); }).catch(() => { return undefined; });
36
+ return deadline(accepted, ctx.options.acceptTimeoutMs, 'Serial acceptance', (error) => {
37
+ ctx.state.fatal = error; ctx.state.active = false; ctx.state.controller.abort(error);
38
+ });
39
+ }
40
+ function enqueue(ctx, chunk) {
41
+ if (!ctx.state.active) { return; }
42
+ const size = chunk.length ?? Buffer.byteLength(String(chunk));
43
+ if (ctx.count >= ctx.options.maxPending || ctx.bytes + size > ctx.options.maxPendingBytes) {
44
+ const error = new Error('Serial receive capacity exceeded; source retired');
45
+ Object.assign(ctx.state, { fatal: error, active: false });
46
+ ctx.state.controller.abort(error);
47
+ ctx.fail(error);
48
+ return;
49
+ }
50
+ const lease = reserve(ctx, size);
51
+ if (!lease) { return; }
52
+ Object.assign(ctx, { count: ctx.count + 1, bytes: ctx.bytes + size });
53
+ ctx.pending = ctx.pending.then(() => {
54
+ if (ctx.state.active && !ctx.failed) {
55
+ return accept(ctx, chunk, lease.release);
56
+ }
57
+ lease.release?.();
58
+ return undefined;
59
+ }).catch(ctx.fail).finally(() => {
60
+ ctx.count -= 1;
61
+ ctx.bytes -= size;
62
+ if (!ctx.count && ctx.state.active && !ctx.failed) { ctx.port.resume?.(); }
63
+ });
64
+ }
65
+ /** Bound data even if a custom driver ignores pause; pause/resume the native stream. */
66
+ export default async function receive(state, port, device, collector, options) {
67
+ const ctx = { state, port, device, collector, options, pending: Promise.resolve(), count: 0, bytes: 0, failed: false };
68
+ const closed = new Promise((resolve) => { state.wake = resolve; });
69
+ ctx.fail = (error) => { ctx.failed = true; report(options, error, device); state.wake?.(); };
70
+ function data(chunk) { enqueue(ctx, chunk); }
71
+ function close() { state.wake?.(); }
72
+ port.on('data', data).on('error', ctx.fail).on('close', close);
73
+ if (!state.active) { state.wake(); }
74
+ await closed;
75
+ port.off('data', data);
76
+ try {
77
+ await closePort(port, options.stopTimeoutMs);
78
+ await deadline(ctx.pending, options.stopTimeoutMs, 'Serial acceptance', (error) => { state.controller.abort(error); });
79
+ } finally { port.off('error', ctx.fail).off('close', close); }
80
+ }
@@ -0,0 +1,87 @@
1
+ import positive from './positive.js';
2
+
3
+ import deadline from './deadline.js';
4
+ import receive, { closePort, report } from './serialReceive.js';
5
+
6
+ async function openPort(device) {
7
+ const { SerialPort } = await import('serialport');
8
+ const port = new SerialPort({ ...device, autoOpen: false });
9
+ await new Promise((resolve, reject) => { port.open((error) => { if (error) { reject(error); } else { resolve(); } }); });
10
+ return port;
11
+ }
12
+ function pause(state, milliseconds) {
13
+ return new Promise((resolve) => { state.wake = resolve; state.timer = setTimeout(resolve, milliseconds); })
14
+ .finally(() => { clearTimeout(state.timer); state.wake = null; });
15
+ }
16
+ async function run(state, device, collector, options) {
17
+ while (state.active) {
18
+ try {
19
+ // Each port has an independent serialized open/close/retry loop.
20
+ const attempt = { timedOut: false };
21
+ const opening = Promise.resolve().then(() => { return options.openPort(device, { signal: state.controller.signal }); });
22
+ void opening.then((late) => {
23
+ if (attempt.timedOut) { return closePort(late, options.stopTimeoutMs); }
24
+ return undefined;
25
+ }).catch((error) => { report(options, error, device); });
26
+ // A timed-out open is uncertain; never open a second handle in this instance.
27
+ // eslint-disable-next-line no-await-in-loop
28
+ const port = await deadline(opening, options.openTimeoutMs, 'Serial open', (error) => { attempt.timedOut = true; state.controller.abort(error); });
29
+ // eslint-disable-next-line no-await-in-loop
30
+ await receive(state, port, device, collector, options);
31
+ } catch (error) {
32
+ if (error.resourceUncertain || error.code === 'OPERATION_TIMEOUT') { Object.assign(state, { fatal: error, active: false }); }
33
+ report(options, error, device);
34
+ }
35
+ if (state.active) {
36
+ // eslint-disable-next-line no-await-in-loop
37
+ await pause(state, options.retryMs);
38
+ }
39
+ }
40
+ }
41
+ function validate(devices, collector) {
42
+ if (!Array.isArray(devices)) { throw new TypeError('devices must be an array'); }
43
+ if (!collector || typeof collector.accept !== 'function') { throw new TypeError('Collector must have an accept() method'); }
44
+ const paths = new Set();
45
+ devices.forEach((device) => {
46
+ if (!device || typeof device.path !== 'string' || !device.path.trim() || paths.has(device.path)) {
47
+ throw new TypeError('invalid or duplicate serial path');
48
+ }
49
+ paths.add(device.path);
50
+ });
51
+ }
52
+ /** Independent serial devices; chunks and device metadata pass through unchanged. */
53
+ export default function serialSource(devices, collector, options = {}) {
54
+ validate(devices, collector);
55
+ const settings = { retryMs: 1000, openTimeoutMs: 5000, acceptTimeoutMs: 10000, stopTimeoutMs: 5000, maxPending: 32, maxPendingBytes: 1048576,
56
+ ...options, openPort: options.openPort ?? openPort };
57
+ ['retryMs', 'acceptTimeoutMs', 'openTimeoutMs', 'stopTimeoutMs', 'maxPending', 'maxPendingBytes'].forEach((key) => { positive(settings[key], key); });
58
+ if (!Number.isSafeInteger(settings.maxPending) || !Number.isSafeInteger(settings.maxPendingBytes)) { throw new TypeError('Serial capacities must be safe integers'); }
59
+ const states = devices.map(() => { return { active: false, task: null, wake: null, timer: null }; });
60
+ let revision = 0;
61
+ let stopping = Promise.resolve();
62
+ return {
63
+ async start() {
64
+ revision += 1;
65
+ const requested = revision;
66
+ await stopping;
67
+ const failed = states.find((state) => { return state.fatal; });
68
+ if (failed) { throw failed.fatal; }
69
+ if (requested === revision) {
70
+ states.forEach((state, index) => {
71
+ if (!state.active) { state.active = true; state.controller = new AbortController(); state.task = run(state, devices[index], collector, settings); }
72
+ });
73
+ }
74
+ },
75
+ stop() {
76
+ revision += 1;
77
+ states.forEach((state) => { state.active = false; state.controller?.abort(new Error('Serial source stopped')); state.wake?.(); });
78
+ stopping = deadline(Promise.all(states.map((state) => { return state.task; })), settings.stopTimeoutMs, 'Serial stop', (error) => {
79
+ states.forEach((state) => { state.fatal = error; });
80
+ }).then(() => {
81
+ const failed = states.find((state) => { return state.fatal; });
82
+ if (failed) { throw failed.fatal; }
83
+ });
84
+ return stopping;
85
+ }
86
+ };
87
+ }