@yarkivaev/scada 2.3.116 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +90 -0
  2. package/charts/scada/templates/processes.yaml +22 -0
  3. package/charts/scada/values.schema.json +12 -0
  4. package/db/migrations/V0009__collector_task_state.sql +6 -0
  5. package/package.json +11 -5
  6. package/src/application/deploy/readDocument.js +4 -1
  7. package/src/application/plantServer.js +1 -1
  8. package/src/application/runtime/additionalAdapters.js +139 -0
  9. package/src/application/runtime/assignment.js +30 -0
  10. package/src/application/runtime/budgetedSink.js +17 -0
  11. package/src/application/runtime/collectorSignals.js +14 -0
  12. package/src/application/runtime/compositeLifecycle.js +49 -0
  13. package/src/application/runtime/createRegistry.js +20 -0
  14. package/src/application/runtime/deadline.js +8 -0
  15. package/src/application/runtime/index.js +10 -0
  16. package/src/application/runtime/manifest.js +38 -0
  17. package/src/application/runtime/pipelineAdapters.js +60 -0
  18. package/src/application/runtime/resolveSecrets.js +12 -0
  19. package/src/application/runtime/resourceClaims.js +21 -0
  20. package/src/application/runtime/runCollector.js +50 -0
  21. package/src/application/runtime/runtime.js +86 -0
  22. package/src/application/runtime/runtimeHealth.js +29 -0
  23. package/src/application/runtime/runtimeServices.js +15 -0
  24. package/src/application/runtime/stateDocument.js +22 -0
  25. package/src/application/runtime/streamPorts.js +42 -0
  26. package/src/application/runtime/streamTask.js +105 -0
  27. package/src/application/runtime/taskBudget.js +31 -0
  28. package/src/application/runtime/taskContext.js +18 -0
  29. package/src/application/runtime/taskQueue.js +21 -0
  30. package/src/application/runtime/taskSlot.js +46 -0
  31. package/src/application/runtime/taskState.js +73 -0
  32. package/src/application/siteCollection.js +14 -0
  33. package/src/application/siteServer.js +7 -6
  34. package/src/application/siteTelemetry.js +1 -1
  35. package/src/application/supervisorSink.js +9 -8
  36. package/src/application/sync/siteSyncBind.js +1 -1
  37. package/src/application/sync/siteSyncTargets.js +1 -1
  38. package/src/bin/collector.js +7 -0
  39. package/src/bin/modbus-to-mqtt.js +11 -9
  40. package/src/infrastructure/ingest/activity/activityTracking.js +15 -21
  41. package/src/infrastructure/ingest/activity/activityTransformer.js +16 -15
  42. package/src/infrastructure/ingest/durableConsume.js +2 -2
  43. package/src/infrastructure/ingest/ingestBatch.js +13 -0
  44. package/src/infrastructure/ingest/modbus/mx210Tcp.js +4 -4
  45. package/src/infrastructure/ingest/modbus/silentStreams.js +33 -8
  46. package/src/infrastructure/ingest/mqtt/metricsTransformer.js +4 -5
  47. package/src/infrastructure/ingest/mqtt/modbusMqtt.js +17 -19
  48. package/src/infrastructure/ingest/mqtt/modbusPublish.js +4 -0
  49. package/src/infrastructure/ingest/mqtt/mqttMetrics.js +17 -21
  50. package/src/infrastructure/ingest/mqtt/registerPublish.js +3 -3
  51. package/src/infrastructure/ingest/pipelines/alertPipeline.js +7 -6
  52. package/src/infrastructure/ingest/pipelines/decisionPipeline.js +7 -6
  53. package/src/infrastructure/ingest/pipelines/segmentPipeline.js +8 -17
  54. package/src/infrastructure/ingest/pipelines/stompOptions.js +7 -0
  55. package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js +10 -11
  56. package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js +10 -6
  57. package/src/infrastructure/ingest/telemetry/foldingSink.js +5 -2
  58. package/src/infrastructure/messaging/stomp/segmentRetags.js +1 -1
  59. package/src/infrastructure/messaging/stomp/userDecisions.js +1 -1
  60. package/src/infrastructure/opcua/certificates.js +11 -0
  61. package/src/infrastructure/opcua/delivery.js +38 -0
  62. package/src/infrastructure/opcua/endpoint.js +12 -2
  63. package/src/infrastructure/opcua/session.js +20 -10
  64. package/src/infrastructure/opcua/source.js +6 -3
  65. package/src/infrastructure/persistence/runtime/taskStatePg.js +24 -0
  66. package/src/infrastructure/streams/README.md +291 -0
  67. package/src/infrastructure/streams/abortable.js +14 -0
  68. package/src/infrastructure/streams/amqpSink/publication.js +46 -0
  69. package/src/infrastructure/streams/amqpSink/session.js +96 -0
  70. package/src/infrastructure/streams/amqpSink.js +47 -0
  71. package/src/infrastructure/streams/amqpSource/session.js +127 -0
  72. package/src/infrastructure/streams/amqpSource.js +97 -0
  73. package/src/infrastructure/streams/brokerClose.js +19 -0
  74. package/src/infrastructure/streams/budgetAccept.js +6 -0
  75. package/src/infrastructure/streams/clickhouseSink.js +48 -0
  76. package/src/infrastructure/streams/deadline.js +1 -0
  77. package/src/infrastructure/streams/follow.js +41 -0
  78. package/src/infrastructure/streams/index.js +15 -0
  79. package/src/infrastructure/streams/lokiSource.js +88 -0
  80. package/src/infrastructure/streams/modbusConnection.js +173 -0
  81. package/src/infrastructure/streams/modbusRtuBusSource.js +124 -0
  82. package/src/infrastructure/streams/modbusSource.js +185 -0
  83. package/src/infrastructure/streams/mqttSink/session.js +51 -0
  84. package/src/infrastructure/streams/mqttSink.js +54 -0
  85. package/src/infrastructure/streams/mqttSource.js +66 -0
  86. package/src/infrastructure/streams/pollingSource.js +1 -0
  87. package/src/infrastructure/streams/positive.js +5 -0
  88. package/src/infrastructure/streams/postgresSink.js +124 -0
  89. package/src/infrastructure/streams/serialReceive.js +80 -0
  90. package/src/infrastructure/streams/serialSource.js +87 -0
  91. package/src/infrastructure/streams/sqliteSink.js +55 -0
  92. package/src/infrastructure/streams/stompBody.js +46 -0
  93. package/src/infrastructure/streams/stompSend.js +89 -0
  94. package/src/infrastructure/streams/stompSource.js +201 -0
  95. package/src/infrastructure/sync/operationSyncIngest.js +1 -1
@@ -15,7 +15,7 @@ const appRoot = path.dirname(fileURLToPath(import.meta.url));
15
15
 
16
16
  /**
17
17
  * Reads deployment environment into a normalized config object.
18
- * `SINK_PIPELINES=false` leaves Postgres and the state HTTP API in this process
18
+ * `COLLECTOR_MODE=external` or `SINK_PIPELINES=false` leaves Postgres and the state HTTP API in this process
19
19
  * and leaves the STOMP consumers to another container.
20
20
  *
21
21
  * @param {NodeJS.ProcessEnv} env - process environment
@@ -41,7 +41,7 @@ export function readDeploymentConfig(env) {
41
41
  retentionDays: parseInt(env.RETENTION_DAYS || '30', 10),
42
42
  migrationsDir: path.join(root, 'db', 'migrations'),
43
43
  requestTimeoutMs: parseRequestTimeoutMs(env.REQUEST_TIMEOUT_MS),
44
- pipelines: env.SINK_PIPELINES !== 'false',
44
+ pipelines: env.COLLECTOR_MODE !== 'external' && env.SINK_PIPELINES !== 'false',
45
45
  pipeline: {
46
46
  prefetch,
47
47
  deadLetter: env.SEGMENTS_DEAD_LETTER_ENABLED === 'true',
@@ -60,6 +60,12 @@ export function readDeploymentConfig(env) {
60
60
  };
61
61
  }
62
62
 
63
+ function startPipelines(cfg, pool) {
64
+ const pipelines = [segmentPipeline(cfg.stomp, cfg.postgres, pool, cfg.pipeline),
65
+ decisionPipeline(cfg.stomp, pool, cfg.pipeline), alertPipeline(cfg.stomp, pool, cfg.pipeline)];
66
+ return Promise.all(pipelines.map((pipeline) => { return pipeline.start(); }));
67
+ }
68
+
63
69
  /**
64
70
  * Wires STOMP pipelines, migrations, and state data access from env config.
65
71
  * HTTP binding uses edgeApi (metrics, checkpoint, retention).
@@ -70,9 +76,6 @@ export function readDeploymentConfig(env) {
70
76
  export default function supervisorSink(env = process.env) {
71
77
  const cfg = readDeploymentConfig(env);
72
78
  const pool = holdClient(new pg.Pool({ connectionString: cfg.postgres }));
73
- const segments = segmentPipeline(cfg.stomp, cfg.postgres, pool, cfg.pipeline);
74
- const decisions = decisionPipeline(cfg.stomp, pool, cfg.pipeline);
75
- const alerts = alertPipeline(cfg.stomp, pool, cfg.pipeline);
76
79
  const metricsEnabled = cfg.sinkDbProfile === 'edge';
77
80
  const dataAccess = stateDataFromPool(pool, { metricsEnabled });
78
81
  return {
@@ -97,9 +100,7 @@ export default function supervisorSink(env = process.env) {
97
100
  await adminPool.end();
98
101
  }
99
102
  if (cfg.pipelines) {
100
- await segments.start();
101
- decisions.start();
102
- alerts.start();
103
+ await startPipelines(cfg, pool);
103
104
  }
104
105
  if (httpServer) {
105
106
  httpServer.start();
@@ -1,4 +1,4 @@
1
- import { clickhouseSink } from '@yarkivaev/source-to-sink';
1
+ import { clickhouseSink } from '../../infrastructure/streams/index.js';
2
2
  import { jobRoutes, jobs } from '@yarkivaev/simple-server';
3
3
  import exportQuery from '../export/exportQuery.js';
4
4
  import scadaClient from '../../infrastructure/client/scadaClient.js';
@@ -1,4 +1,4 @@
1
- import { postgresSink } from '@yarkivaev/source-to-sink';
1
+ import { postgresSink } from '../../infrastructure/streams/index.js';
2
2
  import {
3
3
  alertRow,
4
4
  listedItems,
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env node
2
+ import { createRegistry, runCollector } from '../application/runtime/index.js';
3
+
4
+ try { await runCollector({ registry: createRegistry() }); } catch (error) {
5
+ process.stderr.write(`collector: ${error.name}\n`);
6
+ process.exitCode = 1;
7
+ }
@@ -3,12 +3,7 @@
3
3
  *
4
4
  * Polls ADDRESS or MODBUS_DEVICES and publishes STREAM + MAPPING document paths.
5
5
  */
6
- process.on('unhandledRejection', (err) => {
7
- process.stderr.write(`Unhandled rejection: ${String(err)}\n`);
8
- process.exit(1);
9
- });
10
-
11
- import modbusPublish from '../infrastructure/ingest/mqtt/modbusPublish.js';
6
+ import { createRegistry, runCollector } from '../application/runtime/index.js';
12
7
 
13
8
  /**
14
9
  * Parses a JSON array environment value.
@@ -41,7 +36,7 @@ function integer(text) {
41
36
  }
42
37
 
43
38
  const addresses = jsonList(process.env.ADDRESS);
44
- const pipeline = modbusPublish(process.env.MQTT_URL, {
39
+ const config = {
45
40
  address: integer(process.env.MODBUS_ADDRESS),
46
41
  addresses,
47
42
  clientId: process.env.CLIENT_ID,
@@ -52,6 +47,13 @@ const pipeline = modbusPublish(process.env.MQTT_URL, {
52
47
  streams: jsonList(process.env.STREAM) || [],
53
48
  threshold: parseInt(process.env.CIRCUIT_THRESHOLD || '5', 10),
54
49
  timeout: parseInt(process.env.CIRCUIT_TIMEOUT || '60', 10)
55
- });
50
+ };
56
51
 
57
- pipeline.start();
52
+ try {
53
+ const env = { ...process.env, COLLECTOR_MANIFEST: JSON.stringify({ assignments: [{ taskId: 'modbus', generation: 0, specVersion: 1,
54
+ spec: { adapter: 'modbus', url: { $secret: 'MQTT_URL' }, config } }] }) };
55
+ await runCollector({ registry: createRegistry(), env, argv: [] });
56
+ } catch (error) {
57
+ process.stderr.write(`collector: ${error.name}\n`);
58
+ process.exitCode = 1;
59
+ }
@@ -1,19 +1,14 @@
1
- import {
2
- batch,
3
- circuit,
4
- clickhouseSink,
5
- clock,
6
- lokiSource,
7
- timedBatch
8
- } from '@yarkivaev/source-to-sink';
1
+ import { clock } from '@yarkivaev/source-to-sink';
2
+ import { clickhouseSink, lokiSource } from '../../streams/index.js';
3
+ import ingestBatch from '../ingestBatch.js';
9
4
  import activityCodec from './activityTransformer.js';
10
5
 
11
6
  /**
12
7
  * Pipeline for streaming Loki HTTP access logs to ClickHouse.
13
8
  *
14
- * Polls Loki for log entries matching the query and batches them
15
- * before inserting into ClickHouse `activity.http_access` table.
16
- * Uses circuit breaker for failure isolation.
9
+ * Polls Loki and durably inserts each accepted log into ClickHouse
10
+ * before advancing the serial source cursor.
11
+ * Bounds pending records and bytes and waits for durable writes.
17
12
  *
18
13
  * @example
19
14
  * const pipeline = activityTracking(
@@ -31,10 +26,10 @@ import activityCodec from './activityTransformer.js';
31
26
  * @param {string} query - LogQL query string
32
27
  * @param {object} config - Pipeline configuration
33
28
  * @param {number} config.poll - Loki polling interval in seconds
34
- * @param {number} config.size - Batch size before flush
29
+ * @param {number} config.size - Legacy option; serial durable acceptance writes one record at a time
35
30
  * @param {number} config.interval - Seconds before time-based flush
36
- * @param {number} config.threshold - Circuit breaker failure threshold
37
- * @param {number} config.timeout - Circuit breaker timeout in seconds
31
+ * @param {number} config.threshold - Legacy option (retained for configuration compatibility)
32
+ * @param {number} config.timeout - Legacy option (use writeTimeoutMs for write deadlines)
38
33
  * @returns {object} Pipeline with start() and stop() methods
39
34
  */
40
35
  export default function activityTracking(loki, clickhouse, query, config) {
@@ -51,24 +46,23 @@ export default function activityTracking(loki, clickhouse, query, config) {
51
46
  throw new Error('Config must be an object');
52
47
  }
53
48
  const clk = clock();
54
- const breaker = circuit(config.threshold, config.timeout, clk);
55
49
  const sink = clickhouseSink(clickhouse, 'activity.http_access');
56
- const collector = timedBatch(batch(sink, config.size, breaker), config.interval);
50
+ const collector = ingestBatch(sink, { ...config, size: 1 });
57
51
  const transformer = activityCodec(collector);
58
- const source = lokiSource(loki, query, config.poll, transformer, clk);
52
+ const source = lokiSource(loki, query, config.poll, transformer, clk, { ...config.lokiOptions, budget: config.budget });
59
53
  return {
60
54
  /**
61
55
  * Starts the pipeline.
62
56
  */
63
57
  start() {
64
- source.start();
58
+ return source.start();
65
59
  },
66
60
  /**
67
61
  * Stops the pipeline.
68
62
  */
69
- stop() {
70
- source.stop();
71
- collector.stop();
63
+ async stop() {
64
+ await source.stop();
65
+ try { await collector.stop(); } finally { await sink.stop?.(); }
72
66
  }
73
67
  };
74
68
  }
@@ -22,24 +22,25 @@ export default function activityCodec(collector) {
22
22
  * @param {object} entry - Log entry with ts and line
23
23
  */
24
24
  accept(entry) {
25
+ let log;
25
26
  try {
26
- const log = JSON.parse(entry.line);
27
- if (!log.StartUTC || !log.RequestMethod || !log.RequestPath) {
28
- return;
29
- }
30
- collector.accept({
31
- ts: new Date(log.StartUTC).getTime(),
32
- user: log['request_Remote-User'] || '',
33
- email: log['request_Remote-Email'] || '',
34
- method: log.RequestMethod || '',
35
- path: log.RequestPath || '',
36
- status: parseInt(log.DownstreamStatus, 10) || 0,
37
- duration: parseFloat(log.Duration) / 1e9 || 0,
38
- ip: log.ClientHost || ''
39
- });
27
+ log = JSON.parse(entry.line);
40
28
  } catch {
41
- // Skip malformed log entries
29
+ return undefined;
42
30
  }
31
+ if (!log || !log.StartUTC || !log.RequestMethod || !log.RequestPath) {
32
+ return undefined;
33
+ }
34
+ return collector.accept({
35
+ ts: new Date(log.StartUTC).getTime(),
36
+ user: log['request_Remote-User'] || '',
37
+ email: log['request_Remote-Email'] || '',
38
+ method: log.RequestMethod || '',
39
+ path: log.RequestPath || '',
40
+ status: parseInt(log.DownstreamStatus, 10) || 0,
41
+ duration: parseFloat(log.Duration) / 1e9 || 0,
42
+ ip: log.ClientHost || ''
43
+ });
43
44
  }
44
45
  };
45
46
  }
@@ -1,2 +1,2 @@
1
- // Compatibility facade; transport lifecycle belongs to source-to-sink.
2
- export { amqpSession as default } from '@yarkivaev/source-to-sink';
1
+ // Compatibility facade; transport lifecycle is owned by the scada stream adapters.
2
+ export { amqpSession as default } from '../streams/index.js';
@@ -0,0 +1,13 @@
1
+ import { boundedBatch } from '@yarkivaev/source-to-sink';
2
+
3
+ /** Shared production limits; acceptance resolves only after the sink write. */
4
+ export default function ingestBatch(sink, config = {}) {
5
+ return boundedBatch(sink, {
6
+ ...(config.budget ? { budget: config.budget } : {}),
7
+ size: config.size ?? 100,
8
+ maxRecords: config.maxRecords ?? 1000,
9
+ maxBytes: config.maxBytes ?? 1048576,
10
+ maxWaitMs: config.maxWaitMs ?? (config.interval ?? 5) * 1000,
11
+ writeTimeoutMs: config.writeTimeoutMs ?? 10000
12
+ });
13
+ }
@@ -1,4 +1,4 @@
1
- import { modbusSource } from '@yarkivaev/source-to-sink';
1
+ import { modbusSource } from '../../streams/index.js';
2
2
 
3
3
  const layout = [
4
4
  { offset: 0, channel: 'AI1' },
@@ -34,13 +34,13 @@ export function mx210Metrics(device, collector) {
34
34
  return {
35
35
  accept(registers) {
36
36
  const ts = Date.now();
37
- for (const entry of layout) {
38
- collector.accept({
37
+ return Promise.all(layout.map((entry) => {
38
+ return collector.accept({
39
39
  topic: `MX210/${device}/GET/${entry.channel}/VALUE`,
40
40
  ts,
41
41
  value: decode(registers[entry.offset], registers[entry.offset + 1])
42
42
  });
43
- }
43
+ }));
44
44
  }
45
45
  };
46
46
  }
@@ -1,5 +1,20 @@
1
1
  import { clock, intervalSource } from '@yarkivaev/source-to-sink';
2
2
 
3
+ function track(state, pending) {
4
+ if (!pending || typeof pending.then !== 'function') { return; }
5
+ const task = pending.catch((error) => {
6
+ state.failure = error;
7
+ try { state.onError?.(error); } catch {
8
+ // Reporting must not turn an observed source failure into an unhandled rejection.
9
+ }
10
+ })
11
+ .finally(() => { state.pending.delete(task); });
12
+ state.pending.add(task);
13
+ }
14
+ async function drained(state) {
15
+ await Promise.all(state.pending);
16
+ if (state.failure) { throw state.failure; }
17
+ }
3
18
  /**
4
19
  * Whether a stream is past the silence budget.
5
20
  *
@@ -27,8 +42,12 @@ function stopOne(state, name) {
27
42
  if (!state.live.has(name)) {
28
43
  return;
29
44
  }
30
- state.held.get(name).stop();
31
- state.live.delete(name);
45
+ if (state.closing.has(name)) { return; }
46
+ const pending = state.held.get(name).stop();
47
+ if (pending && typeof pending.then === 'function') {
48
+ state.closing.add(name);
49
+ track(state, pending.finally(() => { state.live.delete(name); state.closing.delete(name); }));
50
+ } else { state.live.delete(name); }
32
51
  }
33
52
 
34
53
  /**
@@ -47,7 +66,7 @@ function startOne(state, source) {
47
66
  poll = source.open(state.collector, state.clk, state.intervalSec);
48
67
  state.held.set(name, poll);
49
68
  }
50
- poll.start();
69
+ track(state, poll.start());
51
70
  state.live.add(name);
52
71
  }
53
72
 
@@ -88,6 +107,10 @@ function gateState(config) {
88
107
  lastSeen: new Map(),
89
108
  held: new Map(),
90
109
  live: new Set(),
110
+ closing: new Set(),
111
+ pending: new Set(),
112
+ failure: null,
113
+ onError: config.onError,
91
114
  collector: undefined,
92
115
  running: false
93
116
  };
@@ -129,22 +152,24 @@ export default function silentStreams(config) {
129
152
  bind(sink) {
130
153
  state.collector = {
131
154
  accept(record) {
132
- sink.write([record]);
155
+ return sink.write([record]);
133
156
  }
134
157
  };
135
158
  },
136
159
  start() {
137
- if (state.running) { return; }
160
+ if (state.running) { return drained(state); }
138
161
  state.running = true;
139
162
  runPulse();
140
- void ticker.start();
163
+ track(state, ticker.start());
164
+ return drained(state);
141
165
  },
142
- stop() {
166
+ async stop() {
143
167
  state.running = false;
144
- void ticker.stop();
168
+ track(state, ticker.stop());
145
169
  for (const name of [...state.live]) {
146
170
  stopOne(state, name);
147
171
  }
172
+ await drained(state);
148
173
  state.held.clear();
149
174
  },
150
175
  pulse: runPulse
@@ -30,25 +30,24 @@ export default function metricsCodec(collector, log = console) {
30
30
  parsed = JSON.parse(raw.payload);
31
31
  } catch {
32
32
  log.warn(`Cannot resolve payload on ${raw.topic}: not valid JSON`);
33
- return;
33
+ return undefined;
34
34
  }
35
35
  if (typeof parsed === 'number') {
36
- collector.accept({
36
+ return collector.accept({
37
37
  topic: raw.topic,
38
38
  ts: Date.now(),
39
39
  value: parsed
40
40
  });
41
- return;
42
41
  }
43
42
  if (typeof parsed === 'object' && parsed !== null && typeof parsed.value === 'number') {
44
- collector.accept({
43
+ return collector.accept({
45
44
  topic: raw.topic,
46
45
  ts: parsed.ts || Date.now(),
47
46
  value: parsed.value
48
47
  });
49
- return;
50
48
  }
51
49
  log.warn(`Cannot resolve payload on ${raw.topic}: no numeric value found`);
50
+ return undefined;
52
51
  }
53
52
  };
54
53
  }
@@ -1,12 +1,6 @@
1
- import {
2
- batch,
3
- circuit,
4
- clock,
5
- modbusRtuBusSource,
6
- modbusRtuSource,
7
- modbusSource,
8
- mqttSink
9
- } from '@yarkivaev/source-to-sink';
1
+ import { clock } from '@yarkivaev/source-to-sink';
2
+ import { modbusRtuBusSource, modbusRtuSource, modbusSource, mqttSink } from '../../streams/index.js';
3
+ import ingestBatch from '../ingestBatch.js';
10
4
  import { parseModbusDeviceSpec } from './modbusDeviceSpec.js';
11
5
 
12
6
  /**
@@ -181,24 +175,28 @@ function buildSources(devices, config, collector, clk) {
181
175
  export default function modbusMqtt(mqtt, devices, config) {
182
176
  assertModbusMqttArgs(mqtt, devices, config);
183
177
  const clk = clock();
184
- const breaker = circuit(config.threshold, config.timeout, clk);
185
178
  const sink = mqttSink(mqtt, {
186
179
  clientId: config.clientId || 'scada-modbus',
187
180
  qos: 1
188
181
  });
189
- const collector = batch(sink, 5, breaker);
182
+ const collector = ingestBatch(sink, { ...config, size: config.size ?? 5 });
190
183
  const sources = buildSources(devices, config, collector, clk);
191
184
  return {
192
- start() {
193
- sink.start();
194
- for (const source of sources) {
195
- source.start();
185
+ async start() {
186
+ await sink.start();
187
+ try {
188
+ await Promise.all(sources.map((source) => { return source.start(); }));
189
+ } catch (error) {
190
+ await Promise.allSettled(sources.map((source) => { return source.stop(); }));
191
+ await sink.stop();
192
+ throw error;
196
193
  }
197
194
  },
198
- stop() {
199
- for (const source of sources) {
200
- source.stop();
201
- }
195
+ async stop() {
196
+ try {
197
+ await Promise.all(sources.map((source) => { return source.stop(); }));
198
+ await collector.stop();
199
+ } finally { await sink.stop(); }
202
200
  }
203
201
  };
204
202
  }
@@ -260,6 +260,10 @@ export default function modbusPublish(mqtt, config) {
260
260
  return [row.name, plans[index]];
261
261
  }));
262
262
  return modbusMqtt(mqtt, specs, {
263
+ budget: config.budget,
264
+ maxRecords: config.maxRecords,
265
+ maxBytes: config.maxBytes,
266
+ writeTimeoutMs: config.writeTimeoutMs,
263
267
  address: frame.address,
264
268
  clientId: config.clientId || 'scada-modbus',
265
269
  count: frame.count,
@@ -1,21 +1,16 @@
1
- import {
2
- batch,
3
- circuit,
4
- clock,
5
- mqttSource,
6
- timedBatch
7
- } from '@yarkivaev/source-to-sink';
1
+ import { mqttSource } from '../../streams/index.js';
2
+ import ingestBatch from '../ingestBatch.js';
8
3
  import metricsCodec from './metricsTransformer.js';
9
4
 
10
5
  /**
11
6
  * Pipeline for streaming MQTT sensor data to a storage sink.
12
7
  *
13
- * Subscribes to MQTT topic and batches messages before writing
14
- * to the provided sink. Uses circuit breaker for failure isolation
15
- * and time-based flushing for low-volume periods.
8
+ * Subscribes to MQTT and writes each message before acknowledging it.
9
+ * MQTT.js serializes acknowledgement, so acceptance cannot wait for a larger batch.
10
+ * Bounds queued records and bytes and enforces a durable-write deadline.
16
11
  *
17
12
  * @example
18
- * import { clickhouseSink } from '@yarkivaev/source-to-sink';
13
+ * import { clickhouseSink } from '../../streams/index.js';
19
14
  * const sink = clickhouseSink('http://localhost:8123', 'scada.metrics');
20
15
  * const pipeline = mqttMetrics(
21
16
  * 'mqtt://localhost:1883',
@@ -31,10 +26,10 @@ import metricsCodec from './metricsTransformer.js';
31
26
  * @param {object} sink - Sink with write(records) method
32
27
  * @param {string} topic - MQTT topic pattern to subscribe
33
28
  * @param {object} config - Pipeline configuration
34
- * @param {number} config.size - Batch size before flush
29
+ * @param {number} config.size - Legacy option; serial durable acceptance writes one record at a time
35
30
  * @param {number} config.interval - Seconds before time-based flush
36
- * @param {number} config.threshold - Circuit breaker failure threshold
37
- * @param {number} config.timeout - Circuit breaker timeout in seconds
31
+ * @param {number} config.threshold - Legacy option (retained for configuration compatibility)
32
+ * @param {number} config.timeout - Legacy option (use writeTimeoutMs for write deadlines)
38
33
  * @param {string} [config.clientId] - MQTT client ID for persistent sessions
39
34
  * @param {number} [config.sessionExpiryInterval] - Session expiry in seconds (default 3600)
40
35
  * @returns {object} Pipeline with start() and stop() methods
@@ -52,11 +47,12 @@ export default function mqttMetrics(mqtt, sink, topic, config) {
52
47
  if (!config || typeof config !== 'object') {
53
48
  throw new Error('Config must be an object');
54
49
  }
55
- const clk = clock();
56
- const breaker = circuit(config.threshold, config.timeout, clk);
57
- const collector = timedBatch(batch(sink, config.size, breaker), config.interval);
50
+ const collector = ingestBatch(sink, { ...config, size: 1 });
58
51
  const transformer = metricsCodec(collector);
59
52
  const source = mqttSource(mqtt, topic, transformer, {
53
+ budget: config.budget,
54
+ prefetch: config.prefetch,
55
+ maxPacketBytes: config.maxBytes,
60
56
  clientId: config.clientId,
61
57
  sessionExpiryInterval: config.sessionExpiryInterval
62
58
  });
@@ -65,14 +61,14 @@ export default function mqttMetrics(mqtt, sink, topic, config) {
65
61
  * Starts the pipeline.
66
62
  */
67
63
  start() {
68
- source.start();
64
+ return source.start();
69
65
  },
70
66
  /**
71
67
  * Stops the pipeline.
72
68
  */
73
- stop() {
74
- source.stop();
75
- collector.stop();
69
+ async stop() {
70
+ await source.stop();
71
+ await collector.stop();
76
72
  }
77
73
  };
78
74
  }
@@ -70,15 +70,15 @@ export default function registerPublish(plans, origin, collector) {
70
70
  * @param {Array<number>} registers - uint16 window
71
71
  */
72
72
  accept(registers) {
73
- rows.forEach((item) => {
74
- collector.accept({
73
+ return Promise.all(rows.map((item) => {
74
+ return collector.accept({
75
75
  payload: JSON.stringify({
76
76
  ts: Date.now(),
77
77
  value: decodePoint(item.point, origin, registers)
78
78
  }),
79
79
  topic: item.topic
80
80
  });
81
- });
81
+ }));
82
82
  }
83
83
  };
84
84
  }
@@ -1,4 +1,5 @@
1
- import { stompSource } from '@yarkivaev/source-to-sink';
1
+ import stompOptions from './stompOptions.js';
2
+ import { stompSource } from '../../streams/index.js';
2
3
  import alertCodec from '../codecs/alertCodec.js';
3
4
  import alertSink from '../sinks/alertSink.js';
4
5
 
@@ -29,20 +30,20 @@ export default function alertPipeline(stomp, pool, config) {
29
30
  const sink = alertSink(pool);
30
31
  const codec = alertCodec(sink, config.translations || {});
31
32
  const source = stompSource(stomp, '/exchange/scada.alerts', codec,
32
- { login: config.login, passcode: config.passcode, host: config.host });
33
+ stompOptions(config));
33
34
  return {
34
35
  /**
35
36
  * Starts the pipeline.
36
37
  */
37
38
  start() {
38
- source.start();
39
+ return source.start();
39
40
  },
40
41
  /**
41
42
  * Stops the pipeline.
42
43
  */
43
- stop() {
44
- source.stop();
45
- sink.stop();
44
+ async stop() {
45
+ await source.stop();
46
+ await sink.stop();
46
47
  }
47
48
  };
48
49
  }
@@ -1,4 +1,5 @@
1
- import { stompSource } from '@yarkivaev/source-to-sink';
1
+ import stompOptions from './stompOptions.js';
2
+ import { stompSource } from '../../streams/index.js';
2
3
  import userDecisionCodec from '../codecs/userDecisionCodec.js';
3
4
  import userDecisionSink from '../sinks/userDecisionSink.js';
4
5
 
@@ -26,20 +27,20 @@ export default function decisionPipeline(stomp, pool, config) {
26
27
  const sink = userDecisionSink(pool);
27
28
  const codec = userDecisionCodec(sink);
28
29
  const source = stompSource(stomp, '/exchange/scada.user_decisions', codec,
29
- { login: config.login, passcode: config.passcode, host: config.host });
30
+ stompOptions(config));
30
31
  return {
31
32
  /**
32
33
  * Starts consuming user decisions from RabbitMQ.
33
34
  */
34
35
  start() {
35
- source.start();
36
+ return source.start();
36
37
  },
37
38
  /**
38
39
  * Stops consuming and releases resources.
39
40
  */
40
- stop() {
41
- source.stop();
42
- pool.end();
41
+ async stop() {
42
+ await source.stop();
43
+ await pool.end();
43
44
  }
44
45
  };
45
46
  }
@@ -1,7 +1,5 @@
1
- import {
2
- postgresSink,
3
- stompSource
4
- } from '@yarkivaev/source-to-sink';
1
+ import stompOptions from './stompOptions.js';
2
+ import { postgresSink, stompSource } from '../../streams/index.js';
5
3
  import segmentDispatch from '../../../domain/segment/dispatch.js';
6
4
  import segmentCoalesce from '../../../domain/segment/segmentCoalesce.js';
7
5
  import silenceBudget from '../../../domain/segment/silenceBudget.js';
@@ -64,14 +62,7 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
64
62
  interval: config.interval
65
63
  });
66
64
  const destination = config.segmentsDestination || segmentsIngestDestination;
67
- const source = stompSource(stomp, destination, coalesce, {
68
- login: config.login,
69
- passcode: config.passcode,
70
- host: config.host,
71
- serial: false,
72
- manualAck: true,
73
- prefetch: config.prefetch
74
- });
65
+ const source = stompSource(stomp, destination, coalesce, stompOptions(config, { serial: false, manualAck: true }));
75
66
  const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
76
67
  intervalMs: (config.poll || 5) * 1000
77
68
  });
@@ -82,13 +73,13 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
82
73
  void repair.catch((error) => {
83
74
  processingErrorLog('abut_segments', error, { budgetSeconds: budget });
84
75
  });
85
- source.start();
76
+ await source.start();
86
77
  await silence.start();
87
78
  },
88
- stop() {
89
- silence.stop();
90
- source.stop();
91
- pool.end();
79
+ async stop() {
80
+ await silence.stop();
81
+ await source.stop();
82
+ await pool.end();
92
83
  }
93
84
  };
94
85
  }