@yarkivaev/scada 2.3.116 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +90 -0
  2. package/charts/scada/templates/processes.yaml +22 -0
  3. package/charts/scada/values.schema.json +12 -0
  4. package/db/migrations/V0009__collector_task_state.sql +6 -0
  5. package/package.json +11 -5
  6. package/src/application/deploy/readDocument.js +4 -1
  7. package/src/application/plantServer.js +1 -1
  8. package/src/application/runtime/additionalAdapters.js +139 -0
  9. package/src/application/runtime/assignment.js +30 -0
  10. package/src/application/runtime/budgetedSink.js +17 -0
  11. package/src/application/runtime/collectorSignals.js +14 -0
  12. package/src/application/runtime/compositeLifecycle.js +49 -0
  13. package/src/application/runtime/createRegistry.js +20 -0
  14. package/src/application/runtime/deadline.js +8 -0
  15. package/src/application/runtime/index.js +10 -0
  16. package/src/application/runtime/manifest.js +38 -0
  17. package/src/application/runtime/pipelineAdapters.js +60 -0
  18. package/src/application/runtime/resolveSecrets.js +12 -0
  19. package/src/application/runtime/resourceClaims.js +21 -0
  20. package/src/application/runtime/runCollector.js +50 -0
  21. package/src/application/runtime/runtime.js +86 -0
  22. package/src/application/runtime/runtimeHealth.js +29 -0
  23. package/src/application/runtime/runtimeServices.js +15 -0
  24. package/src/application/runtime/stateDocument.js +22 -0
  25. package/src/application/runtime/streamPorts.js +42 -0
  26. package/src/application/runtime/streamTask.js +105 -0
  27. package/src/application/runtime/taskBudget.js +31 -0
  28. package/src/application/runtime/taskContext.js +18 -0
  29. package/src/application/runtime/taskQueue.js +21 -0
  30. package/src/application/runtime/taskSlot.js +46 -0
  31. package/src/application/runtime/taskState.js +73 -0
  32. package/src/application/siteCollection.js +14 -0
  33. package/src/application/siteServer.js +7 -6
  34. package/src/application/siteTelemetry.js +1 -1
  35. package/src/application/supervisorSink.js +9 -8
  36. package/src/application/sync/siteSyncBind.js +1 -1
  37. package/src/application/sync/siteSyncTargets.js +1 -1
  38. package/src/bin/collector.js +7 -0
  39. package/src/bin/modbus-to-mqtt.js +11 -9
  40. package/src/infrastructure/ingest/activity/activityTracking.js +15 -21
  41. package/src/infrastructure/ingest/activity/activityTransformer.js +16 -15
  42. package/src/infrastructure/ingest/durableConsume.js +2 -2
  43. package/src/infrastructure/ingest/ingestBatch.js +13 -0
  44. package/src/infrastructure/ingest/modbus/mx210Tcp.js +4 -4
  45. package/src/infrastructure/ingest/modbus/silentStreams.js +33 -8
  46. package/src/infrastructure/ingest/mqtt/metricsTransformer.js +4 -5
  47. package/src/infrastructure/ingest/mqtt/modbusMqtt.js +17 -19
  48. package/src/infrastructure/ingest/mqtt/modbusPublish.js +4 -0
  49. package/src/infrastructure/ingest/mqtt/mqttMetrics.js +17 -21
  50. package/src/infrastructure/ingest/mqtt/registerPublish.js +3 -3
  51. package/src/infrastructure/ingest/pipelines/alertPipeline.js +7 -6
  52. package/src/infrastructure/ingest/pipelines/decisionPipeline.js +7 -6
  53. package/src/infrastructure/ingest/pipelines/segmentPipeline.js +8 -17
  54. package/src/infrastructure/ingest/pipelines/stompOptions.js +7 -0
  55. package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js +10 -11
  56. package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js +10 -6
  57. package/src/infrastructure/ingest/telemetry/foldingSink.js +5 -2
  58. package/src/infrastructure/messaging/stomp/segmentRetags.js +1 -1
  59. package/src/infrastructure/messaging/stomp/userDecisions.js +1 -1
  60. package/src/infrastructure/opcua/certificates.js +11 -0
  61. package/src/infrastructure/opcua/delivery.js +38 -0
  62. package/src/infrastructure/opcua/endpoint.js +12 -2
  63. package/src/infrastructure/opcua/session.js +20 -10
  64. package/src/infrastructure/opcua/source.js +6 -3
  65. package/src/infrastructure/persistence/runtime/taskStatePg.js +24 -0
  66. package/src/infrastructure/streams/README.md +291 -0
  67. package/src/infrastructure/streams/abortable.js +14 -0
  68. package/src/infrastructure/streams/amqpSink/publication.js +46 -0
  69. package/src/infrastructure/streams/amqpSink/session.js +105 -0
  70. package/src/infrastructure/streams/amqpSink.js +47 -0
  71. package/src/infrastructure/streams/amqpSource/session.js +127 -0
  72. package/src/infrastructure/streams/amqpSource.js +97 -0
  73. package/src/infrastructure/streams/brokerClose.js +19 -0
  74. package/src/infrastructure/streams/budgetAccept.js +6 -0
  75. package/src/infrastructure/streams/clickhouseSink.js +48 -0
  76. package/src/infrastructure/streams/deadline.js +1 -0
  77. package/src/infrastructure/streams/follow.js +41 -0
  78. package/src/infrastructure/streams/index.js +15 -0
  79. package/src/infrastructure/streams/lokiSource.js +88 -0
  80. package/src/infrastructure/streams/modbusConnection.js +173 -0
  81. package/src/infrastructure/streams/modbusRtuBusSource.js +124 -0
  82. package/src/infrastructure/streams/modbusSource.js +185 -0
  83. package/src/infrastructure/streams/mqttSink/session.js +51 -0
  84. package/src/infrastructure/streams/mqttSink.js +54 -0
  85. package/src/infrastructure/streams/mqttSource.js +66 -0
  86. package/src/infrastructure/streams/pollingSource.js +1 -0
  87. package/src/infrastructure/streams/positive.js +5 -0
  88. package/src/infrastructure/streams/postgresSink.js +124 -0
  89. package/src/infrastructure/streams/serialReceive.js +80 -0
  90. package/src/infrastructure/streams/serialSource.js +87 -0
  91. package/src/infrastructure/streams/sqliteSink.js +55 -0
  92. package/src/infrastructure/streams/stompBody.js +46 -0
  93. package/src/infrastructure/streams/stompSend.js +89 -0
  94. package/src/infrastructure/streams/stompSource.js +201 -0
  95. package/src/infrastructure/sync/operationSyncIngest.js +1 -1
@@ -0,0 +1,46 @@
1
+ import { randomUUID } from 'node:crypto';
2
+
3
+ function publishOptions(record) {
4
+ const options = record.options || {};
5
+ if (!options.mandatory) { return options; }
6
+ const headers = { ...options.headers, 'x-source-to-sink-publish-id': randomUUID() };
7
+ return { ...options, headers };
8
+ }
9
+ /** Confirms one send, without replaying an outcome the broker may already have accepted. */
10
+ export default function publication(live, record, timeoutMs, signal) {
11
+ if (!record || typeof record.exchange !== 'string' || typeof record.routingKey !== 'string'
12
+ || !(typeof record.body === 'string' || Buffer.isBuffer(record.body))) {
13
+ return Promise.reject(new TypeError('AMQP record requires exchange, routingKey and string/Buffer body'));
14
+ }
15
+ if (signal?.aborted) { return Promise.reject(signal.reason || new Error('AMQP publish aborted')); }
16
+ const options = publishOptions(record);
17
+ return new Promise((resolve, reject) => {
18
+ const body = Buffer.isBuffer(record.body) ? record.body : Buffer.from(record.body);
19
+ let settled = false;
20
+ function abort() { live.fail(signal.reason || new Error('AMQP publish aborted')); }
21
+ const timer = { handle: null };
22
+ function settle(error) {
23
+ if (!settled) {
24
+ settled = true;
25
+ clearTimeout(timer.handle);
26
+ signal?.removeEventListener('abort', abort);
27
+ live.pending.delete(settle);
28
+ if (options.mandatory) { live.returns.delete(options.headers['x-source-to-sink-publish-id']); }
29
+ if (error) { reject(error); } else { resolve(); }
30
+ }
31
+ }
32
+ timer.handle = setTimeout(() => {
33
+ const error = new Error('AMQP publish confirmation timed out');
34
+ settle(error);
35
+ live.fail(error);
36
+ }, timeoutMs);
37
+ live.pending.add(settle);
38
+ if (options.mandatory) { live.returns.set(options.headers['x-source-to-sink-publish-id'], settle); }
39
+ signal?.addEventListener('abort', abort, { once: true });
40
+ try {
41
+ if (!live.channel.publish(record.exchange, record.routingKey, body, options, settle)) {
42
+ live.blocked = true;
43
+ }
44
+ } catch (error) { settle(error); }
45
+ });
46
+ }
@@ -0,0 +1,105 @@
1
+ import amqp from 'amqplib';
2
+ import brokerClose, { destroyConnection } from '../brokerClose.js';
3
+ import deadline from '../deadline.js';
4
+
5
+ export function report(state, error) {
6
+ try { state.options.onError?.(error); } catch {
7
+ // Diagnostics cannot break recovery.
8
+ }
9
+ }
10
+ export function closeLive(live) {
11
+ if (!live) { return Promise.resolve(); }
12
+ live.closing ??= Promise.resolve().then(async () => {
13
+ live.pending.forEach((settle) => { settle(new Error('AMQP connection closed before confirmation')); });
14
+ await brokerClose([
15
+ () => { return live.channel?.close(); }, () => { return live.connection?.close(); }
16
+ ], live.options.stopTimeoutMs, 'AMQP close', () => { destroyConnection(live.connection); });
17
+ });
18
+ return live.closing;
19
+ }
20
+ function retry(state, epoch) {
21
+ if (state.wanted && state.epoch === epoch && !state.retry) {
22
+ state.retry = setTimeout(() => {
23
+ state.retry = null;
24
+ void state.reopen().catch((error) => { report(state, error); });
25
+ }, state.options.retryMs);
26
+ }
27
+ }
28
+ function drop(state, live, error) {
29
+ if (live.dead) { return; }
30
+ live.dead = true;
31
+ live.pending.forEach((settle) => { settle(error || new Error('AMQP connection closed')); });
32
+ if (state.live === live) { state.live = null; }
33
+ if (error) { report(state, error); }
34
+ void closeLive(live).then(() => { retry(state, live.epoch); }, (failure) => { state.fatal = failure; report(state, failure); });
35
+ }
36
+ function bind(state, live, handle) {
37
+ handle.on('error', (error) => { drop(state, live, error); });
38
+ handle.on('close', () => { drop(state, live); });
39
+ }
40
+ function returned(live, message) {
41
+ const settle = live.returns.get(message.properties?.headers?.['x-source-to-sink-publish-id']);
42
+ const error = new Error(`AMQP publication returned unroutable: ${message.fields?.replyText || 'NO_ROUTE'}`);
43
+ error.code = 'AMQP_UNROUTABLE';
44
+ if (settle) { settle(error); } else { live.fail(error); }
45
+ }
46
+ async function configure(state, live) {
47
+ // This attempt owns live; stop closes it after opening settles and epoch is checked below.
48
+ // eslint-disable-next-line require-atomic-updates
49
+ live.channel = await live.connection.createConfirmChannel();
50
+ bind(state, live, live.channel);
51
+ live.channel.on('return', (message) => { returned(live, message); });
52
+ live.channel.on('drain', () => { live.blocked = false; });
53
+ await Promise.all((state.options.exchanges || []).map((item) => {
54
+ return live.channel.assertExchange(item.name, item.type || 'topic', item.options || {durable: true});
55
+ }));
56
+ }
57
+ async function current(state, live) {
58
+ if (live.epoch !== state.epoch || !state.wanted) {
59
+ await closeLive(live);
60
+ return false;
61
+ }
62
+ return true;
63
+ }
64
+ function keep(state, live) {
65
+ if (live.epoch !== state.epoch || !state.wanted) { return closeLive(live); }
66
+ state.live = live;
67
+ return undefined;
68
+ }
69
+ async function connect(state, epoch) {
70
+ const live = { options: state.options, connection: null, channel: null, pending: new Set(), returns: new Map(), epoch, dead: false, blocked: false,
71
+ fail(error) { drop(state, live, error); } };
72
+ try {
73
+ live.connection = await (state.options.connect || amqp.connect)(state.url, { timeout: state.options.connectTimeoutMs });
74
+ bind(state, live, live.connection);
75
+ if (!await current(state, live)) { return; }
76
+ await configure(state, live);
77
+ if (live.dead) { throw new Error('AMQP session closed during initialization'); }
78
+ await keep(state, live);
79
+ } catch (error) {
80
+ await closeLive(live);
81
+ retry(state, epoch);
82
+ throw error;
83
+ }
84
+ }
85
+ function recover(state, timeout) {
86
+ if (timeout && state.fatal === timeout) {
87
+ state.fatal = null;
88
+ retry(state, state.epoch);
89
+ }
90
+ }
91
+ export function begin(state) {
92
+ if (state.fatal) { return Promise.reject(state.fatal); }
93
+ if (state.live) { return Promise.resolve(); }
94
+ if (state.opening) { return state.opening; }
95
+ if (!state.wanted) { return Promise.resolve(); }
96
+ clearTimeout(state.retry);
97
+ state.retry = null;
98
+ let timeout;
99
+ const attempt = connect(state, state.epoch);
100
+ void attempt.then(() => { recover(state, timeout); }, () => { recover(state, timeout); });
101
+ state.opening = deadline(attempt, state.options.connectTimeoutMs, 'AMQP connect', (error) => {
102
+ timeout = error; state.fatal = error; state.epoch += 1;
103
+ }).finally(() => { state.opening = null; });
104
+ return state.opening;
105
+ }
@@ -0,0 +1,47 @@
1
+ import { begin, closeLive } from './amqpSink/session.js';
2
+ import publication from './amqpSink/publication.js';
3
+
4
+ function settings(url, options) {
5
+ if (typeof url !== 'string' || !/^amqps?:\/\//u.test(url)) { throw new TypeError('AMQP URL is required'); }
6
+ const config = { retryMs: 1000, connectTimeoutMs: 5000, confirmTimeoutMs: 10000, stopTimeoutMs: 5000, maxPending: 1024, ...options };
7
+ ['retryMs', 'connectTimeoutMs', 'confirmTimeoutMs', 'stopTimeoutMs', 'maxPending'].forEach((name) => {
8
+ if (!Number.isFinite(config[name]) || config[name] <= 0) { throw new TypeError(`${name} must be positive`); }
9
+ });
10
+ if (!Number.isInteger(config.maxPending)) { throw new TypeError('maxPending must be an integer'); }
11
+ return config;
12
+ }
13
+ function stop(state) {
14
+ state.request += 1;
15
+ state.epoch += 1;
16
+ state.wanted = false;
17
+ clearTimeout(state.retry);
18
+ state.retry = null;
19
+ const { live } = state;
20
+ state.live = null;
21
+ state.stopped = Promise.all([closeLive(live), state.opening?.catch(() => { return undefined; })]);
22
+ return state.stopped;
23
+ }
24
+ /** AMQP record sink with confirms and independent connection recovery; never automatically replays sends. */
25
+ export default function amqpSink(url, options = {}) {
26
+ const state = {url, options: settings(url, options), wanted: false, epoch: 0, request: 0,
27
+ live: null, opening: null, retry: null, stopped: Promise.resolve(), reopen() { return begin(state); }};
28
+ return {
29
+ active() { return Boolean(state.live); },
30
+ start() {
31
+ state.wanted = true;
32
+ state.request += 1;
33
+ const { request } = state;
34
+ return state.stopped.then(() => { if (request === state.request) { return begin(state); } return undefined; });
35
+ },
36
+ stop() { return stop(state); },
37
+ write(records, { signal } = {}) {
38
+ if (!Array.isArray(records)) { return Promise.reject(new TypeError('AMQP records must be an array')); }
39
+ if (!state.live) { return Promise.reject(new Error('AMQP sink is not connected')); }
40
+ const { live } = state;
41
+ if (live.blocked || live.pending.size + records.length > state.options.maxPending) {
42
+ return Promise.reject(new Error('AMQP sink backpressure: retry after confirmations or drain'));
43
+ }
44
+ return Promise.all(records.map((record) => { return publication(live, record, state.options.confirmTimeoutMs, signal); }));
45
+ }
46
+ };
47
+ }
@@ -0,0 +1,127 @@
1
+ // Each session is mutated only by its generation; disposal waits for subscription.
2
+ /* eslint-disable require-atomic-updates */
3
+ import brokerClose, { destroyConnection } from '../brokerClose.js';
4
+ import deadline from '../deadline.js';
5
+ import positive from '../positive.js';
6
+
7
+ async function connectBroker(url) {
8
+ const { default: amqp } = await import('amqplib');
9
+ return amqp.connect(url, { heartbeat: 10 });
10
+ }
11
+ function reportFailure(error) {
12
+ process.stderr.write(`amqp consume failed ${error.message}\n`);
13
+ }
14
+ async function closeLive(live, ctx) {
15
+ try {
16
+ await brokerClose([
17
+ () => { return live.tag ? live.channel.cancel(live.tag) : undefined; },
18
+ () => { return ctx.drain?.(live.channel); },
19
+ () => { return live.channel?.close(); },
20
+ () => { return live.connection?.close(); }
21
+ ], ctx.stopTimeoutMs, 'AMQP cleanup', () => { destroyConnection(live.connection); });
22
+ } catch (error) { ctx.state.fatal = error; throw error; }
23
+ }
24
+ function dispose(live, ctx) {
25
+ if (!live) { return Promise.resolve(); }
26
+ if (!live.closing) { live.closing = closeLive(live, ctx); }
27
+ return live.closing;
28
+ }
29
+
30
+ function current(ctx, generation) {
31
+ return !ctx.state.fatal && ctx.state.running && ctx.state.generation === generation;
32
+ }
33
+ function retry(ctx, generation) {
34
+ if (!current(ctx, generation) || ctx.state.timer !== undefined) { return; }
35
+ const pause = ctx.state.delay;
36
+ ctx.state.delay = Math.min(pause * 2, 30000);
37
+ ctx.state.timer = ctx.schedule(() => {
38
+ ctx.state.timer = undefined;
39
+ ctx.state.opening = deadline(ctx.open(generation), ctx.connectTimeoutMs, 'AMQP connect', (error) => {
40
+ ctx.state.fatal = error; ctx.state.running = false; ctx.state.generation += 1;
41
+ });
42
+ void ctx.state.opening.catch(ctx.report);
43
+ return ctx.state.opening;
44
+ }, pause);
45
+ }
46
+ function bind(ctx, generation, live, emitter) {
47
+ function lost(error) {
48
+ if (live.lost || !current(ctx, generation)) { return; }
49
+ live.lost = true;
50
+ if (error) { ctx.report(error); }
51
+ ctx.state.cleanup = live.ready.then(() => { return dispose(live, ctx); }).then(() => { retry(ctx, generation); });
52
+ void ctx.state.cleanup.catch(ctx.report);
53
+ }
54
+ emitter.on('error', lost);
55
+ emitter.on('close', lost);
56
+ }
57
+ async function subscribe(ctx, generation, live) {
58
+ if (!current(ctx, generation) || live.lost) { return; }
59
+ live.channel = await live.connection.createChannel();
60
+ bind(ctx, generation, live, live.channel);
61
+ if (!current(ctx, generation) || live.lost) { return; }
62
+ live.tag = await ctx.subscribe(live.channel);
63
+ }
64
+ async function open(ctx, generation) {
65
+ let complete;
66
+ const live = { closed: false, ready: new Promise((resolve) => { complete = resolve; }) };
67
+ try {
68
+ live.connection = await ctx.connect(ctx.url);
69
+ bind(ctx, generation, live, live.connection);
70
+ await subscribe(ctx, generation, live);
71
+ if (!current(ctx, generation) || live.lost) { await dispose(live, ctx); return; }
72
+ ctx.state.live = live;
73
+ ctx.state.delay = 1000;
74
+ } catch (error) {
75
+ ctx.report(error);
76
+ await dispose(live, ctx);
77
+ retry(ctx, generation);
78
+ } finally { complete(); }
79
+ }
80
+ async function begin(ctx) {
81
+ if (ctx.state.fatal) { throw ctx.state.fatal; }
82
+ if (ctx.state.running) { await ctx.state.opening; return; }
83
+ ctx.state.running = true;
84
+ ctx.state.generation += 1;
85
+ const { generation } = ctx.state;
86
+ await ctx.state.stopping;
87
+ await ctx.state.cleanup;
88
+ if (current(ctx, generation)) {
89
+ ctx.state.opening = deadline(ctx.open(generation), ctx.connectTimeoutMs, 'AMQP connect', (error) => {
90
+ ctx.state.fatal = error; ctx.state.running = false; ctx.state.generation += 1;
91
+ });
92
+ await ctx.state.opening;
93
+ }
94
+ }
95
+ async function end(ctx) {
96
+ ctx.state.running = false;
97
+ ctx.state.generation += 1;
98
+ if (ctx.state.timer !== undefined) { ctx.clear(ctx.state.timer); ctx.state.timer = undefined; }
99
+ await ctx.state.opening;
100
+ await ctx.state.cleanup;
101
+ await dispose(ctx.state.live, ctx);
102
+ ctx.state.live = undefined;
103
+ }
104
+ /** Shared AMQP session lifecycle; subscribe returns a consumer tag. */
105
+ export default function amqpSession(url, subscribeChannel, options = {}) {
106
+ positive(options.stopTimeoutMs ?? 5000, 'stopTimeoutMs');
107
+ positive(options.connectTimeoutMs ?? 5000, 'connectTimeoutMs');
108
+ const ctx = {
109
+ stopTimeoutMs: options.stopTimeoutMs ?? 5000, connectTimeoutMs: options.connectTimeoutMs ?? 5000,
110
+ url, subscribe: subscribeChannel, drain: options.drain,
111
+ connect: options.connect || connectBroker,
112
+ schedule: options.schedule || setTimeout,
113
+ clear: options.clear || clearTimeout,
114
+ report: options.onError || options.report || reportFailure,
115
+ state: { generation: 0, running: false, delay: 1000 },
116
+ open(generation) { return open(ctx, generation); }
117
+ };
118
+ return {
119
+ start() { return begin(ctx); },
120
+ stop() {
121
+ ctx.state.stopping = end(ctx);
122
+ return ctx.state.stopping;
123
+ },
124
+ generation() { return ctx.state.generation; },
125
+ active() { return !ctx.state.fatal && ctx.state.running; }
126
+ };
127
+ }
@@ -0,0 +1,97 @@
1
+ import amqpSession from './amqpSource/session.js';
2
+ import budgetAccept from './budgetAccept.js';
3
+
4
+ function capacity(options) {
5
+ const settings = { prefetch: 32, maxMessageBytes: 1048576, ...options };
6
+ settings.maxPendingBytes ??= settings.prefetch * settings.maxMessageBytes;
7
+ ['prefetch', 'maxMessageBytes', 'maxPendingBytes'].forEach((name) => {
8
+ if (!Number.isSafeInteger(settings[name]) || settings[name] < 1) { throw new TypeError(`${name} must be a positive safe integer`); }
9
+ });
10
+ return settings;
11
+ }
12
+
13
+ function deliveryFor(channel, message, options) {
14
+ let settled = false;
15
+ return {
16
+ content: message.content, fields: message.fields, properties: message.properties,
17
+ ack() {
18
+ if (!settled && options.active()) { channel.ack(message); settled = true; }
19
+ },
20
+ reject(requeue = true) {
21
+ if (!settled && options.active()) { channel.nack(message, false, requeue); settled = true; }
22
+ }
23
+ };
24
+ }
25
+ function reportFailure(options, error) {
26
+ const report = options.onError || options.report;
27
+ try {
28
+ if (report) { report(error); }
29
+ else { process.stderr.write(`amqp delivery failed ${error.message}\n`); }
30
+ } catch {
31
+ // Error reporting must not reject a broker callback.
32
+ }
33
+ }
34
+ function rejectDelivery(delivery, options, error) {
35
+ try { delivery.reject(true); }
36
+ catch (settlementError) { reportFailure(options, settlementError); }
37
+ reportFailure(options, error);
38
+ }
39
+ function consume(channel, collector, options) {
40
+ return async (message) => {
41
+ if (!message) { channel.emit('close'); return; }
42
+ const delivery = deliveryFor(channel, message, options);
43
+ try {
44
+ await budgetAccept(collector, delivery, message.content.length, options);
45
+ if (options.ackMode !== 'manual') { delivery.ack(); }
46
+ } catch (error) {
47
+ rejectDelivery(delivery, options, error);
48
+ }
49
+ };
50
+ }
51
+ function trackedConsume(channel, collector, options, pending) {
52
+ const accept = consume(channel, collector, options);
53
+ const state = { bytes: 0 };
54
+ return (message) => {
55
+ if (!options.active()) { return Promise.resolve(); }
56
+ const bytes = message?.content?.length ?? 0;
57
+ if (bytes > options.maxMessageBytes || state.bytes + bytes > options.maxPendingBytes) {
58
+ try { channel.nack(message, false, false); } catch (error) { reportFailure(options, error); }
59
+ reportFailure(options, new Error('AMQP payload capacity exceeded; rejected without requeue'));
60
+ return Promise.resolve();
61
+ }
62
+ state.bytes += bytes;
63
+ const delivery = accept(message);
64
+ pending.add(delivery);
65
+ return delivery.finally(() => { state.bytes -= bytes; pending.delete(delivery); });
66
+ };
67
+ }
68
+ async function topology(channel, queue, options) {
69
+ await Promise.all((options.exchanges || []).map((exchange) => {
70
+ return channel.assertExchange(exchange.name, exchange.type, exchange.options);
71
+ }));
72
+ const declared = await channel.assertQueue(queue, options.queueOptions || { durable: true });
73
+ const name = declared?.queue || queue;
74
+ await Promise.all((options.bindings || []).map((binding) => {
75
+ return channel.bindQueue(name, binding.exchange, binding.pattern, binding.args);
76
+ }));
77
+ await channel.prefetch(options.prefetch || 32);
78
+ return name;
79
+ }
80
+ /** Delivers AMQP records, acknowledging only after collector acceptance. */
81
+ export default function amqpSource(url, queue, collector, options = {}) {
82
+ if (!collector || typeof collector.accept !== 'function') { throw new Error('Collector must have an accept() method'); }
83
+ const settings = capacity(options);
84
+ const pending = new Set();
85
+ const state = {};
86
+ const session = amqpSession(url, async (channel) => {
87
+ const generation = state.session.generation();
88
+ const config = { ...settings, active() {
89
+ return state.session.active() && state.session.generation() === generation;
90
+ } };
91
+ const name = await topology(channel, queue, settings);
92
+ const result = await channel.consume(name, trackedConsume(channel, collector, config, pending), { noAck: false });
93
+ return result.consumerTag;
94
+ }, { ...settings, drain() { return Promise.all([...pending]); } });
95
+ state.session = session;
96
+ return session;
97
+ }
@@ -0,0 +1,19 @@
1
+ import deadline from './deadline.js';
2
+
3
+ /** Attempt every cleanup stage; an uncertain stage prevents reuse even if later cleanup succeeds. */
4
+ export default async function brokerClose(actions, timeoutMs, label, destroy) {
5
+ const failures = [];
6
+ for (const action of actions) {
7
+ try {
8
+ // Cleanup order matters, but one hung stage must not skip transport closure.
9
+ // eslint-disable-next-line no-await-in-loop
10
+ await deadline(Promise.resolve().then(action), timeoutMs, label, destroy);
11
+ } catch (error) {
12
+ if (error.name !== 'IllegalOperationError') { failures.push(error); }
13
+ }
14
+ }
15
+ if (failures.length) { throw failures[0]; }
16
+ }
17
+ export function destroyConnection(connection) {
18
+ connection?.connection?.stream?.destroy();
19
+ }
@@ -0,0 +1,6 @@
1
+ /** Budget leases belong to actual acceptance, never to a timeout wrapper. */
2
+ export default async function budgetAccept(collector, record, bytes, options, context) {
3
+ const release = options.budget?.reserve({ bytes, records: 1 });
4
+ try { return await collector.accept(record, context); }
5
+ finally { release?.(); }
6
+ }
@@ -0,0 +1,48 @@
1
+ import { createClient } from '@clickhouse/client';
2
+ import deadline from './deadline.js';
3
+ import positive from './positive.js';
4
+
5
+ /**
6
+ * ClickHouse sink for batch record insertion.
7
+ *
8
+ * Creates a ClickHouse client internally and implements the Sink
9
+ * interface for use with batch collectors.
10
+ *
11
+ * @example
12
+ * const sink = clickhouseSink('http://localhost:8123', 'metrics');
13
+ * await sink.write([{ ts: Date.now(), value: 42 }]);
14
+ *
15
+ * @param {string} url - ClickHouse URL (e.g., 'http://localhost:8123')
16
+ * @param {string} table - Target table name
17
+ * @returns {object} Sink with write(records) method
18
+ */
19
+ export default function clickhouseSink(url, table, options = {}) {
20
+ if (typeof url !== 'string' || url.length === 0) {
21
+ throw new Error('URL must be a non-empty string');
22
+ }
23
+ if (typeof table !== 'string' || table.length === 0) {
24
+ throw new Error('Table must be a non-empty string');
25
+ }
26
+ positive(options.stopTimeoutMs ?? 5000, 'stopTimeoutMs');
27
+ const client = (options.createClient || createClient)({ url });
28
+ let stopping;
29
+ return {
30
+ /**
31
+ * Writes records to ClickHouse table.
32
+ *
33
+ * @param {Array} records - Array of records to insert
34
+ * @returns {Promise} Promise resolving when insert completes
35
+ */
36
+ write(records, { signal } = {}) {
37
+ if (stopping) { return Promise.reject(new Error('ClickHouse sink stopped')); }
38
+ if (signal?.aborted) { return Promise.reject(signal.reason || new Error('ClickHouse write aborted')); }
39
+ // Native ClickHouse API uses snake_case for cancellation.
40
+
41
+ return client.insert({ table, values: records, format: 'JSONEachRow', abort_signal: signal });
42
+ },
43
+ stop() {
44
+ stopping ??= deadline(Promise.resolve().then(() => { return client.close(); }), options.stopTimeoutMs ?? 5000, 'ClickHouse close');
45
+ return stopping;
46
+ }
47
+ };
48
+ }
@@ -0,0 +1 @@
1
+ export { deadline as default } from '@yarkivaev/source-to-sink';
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Continues a delivery chain after a rejected prior step.
3
+ *
4
+ * @param {Promise} pending - prior chain
5
+ * @returns {Promise} settled chain
6
+ */
7
+ function resume(pending) {
8
+ return pending.then((value) => {
9
+ return value;
10
+ }, (error) => {
11
+ return error;
12
+ });
13
+ }
14
+
15
+ /**
16
+ * Chains one arrival: read the body, then start deliver.
17
+ * Serial waits for deliver. Otherwise persist may overlap after enqueue.
18
+ *
19
+ * @param {Promise} pending - prior arrival
20
+ * @param {Function} read - returns a promise for the body
21
+ * @param {Function} deliver - starts persist for one body
22
+ * @param {boolean} serial - wait for persist before the next arrival
23
+ * @returns {Promise} chain
24
+ *
25
+ * @example
26
+ * let pending = Promise.resolve();
27
+ * pending = follow(pending, () => Promise.resolve('a'), write, false);
28
+ */
29
+ export default function follow(pending, read, deliver, serial) {
30
+ return resume(pending).then(async () => {
31
+ const body = await read();
32
+ const run = deliver(body);
33
+ if (serial) {
34
+ return run;
35
+ }
36
+ void run.catch((error) => {
37
+ return error;
38
+ });
39
+ return undefined;
40
+ });
41
+ }
@@ -0,0 +1,15 @@
1
+ /** Protocol and storage adapters composed with source-to-sink primitives. */
2
+ export { default as clickhouseSink } from './clickhouseSink.js';
3
+ export { default as postgresSink } from './postgresSink.js';
4
+ export { default as mqttSource } from './mqttSource.js';
5
+ export { default as stompSource } from './stompSource.js';
6
+ export { default as stompSend } from './stompSend.js';
7
+ export { default as lokiSource } from './lokiSource.js';
8
+ export { default as modbusSource, modbusRtuSource } from './modbusSource.js';
9
+ export { default as modbusRtuBusSource } from './modbusRtuBusSource.js';
10
+ export { default as mqttSink } from './mqttSink.js';
11
+ export { default as sqliteSink } from './sqliteSink.js';
12
+ export { default as amqpSink } from './amqpSink.js';
13
+ export { default as amqpSource } from './amqpSource.js';
14
+ export { default as amqpSession } from './amqpSource/session.js';
15
+ export { default as serialSource } from './serialSource.js';
@@ -0,0 +1,88 @@
1
+ import pollingSource from './pollingSource.js';
2
+ import positive from './positive.js';
3
+
4
+ function config(options) {
5
+ const settings = { requestTimeoutMs: 5000, maxResponseBytes: 1048576, pageLimit: 1000, queryLagMs: 1000, ...options };
6
+ ['requestTimeoutMs', 'maxResponseBytes', 'pageLimit'].forEach((key) => { positive(settings[key], key); });
7
+ if (!Number.isSafeInteger(settings.pageLimit) || settings.pageLimit > 10000 || !Number.isSafeInteger(settings.maxResponseBytes)) {
8
+ throw new TypeError('Invalid Loki response capacity');
9
+ }
10
+ if (!Number.isSafeInteger(settings.queryLagMs) || settings.queryLagMs < 0) { throw new TypeError('Invalid Loki queryLagMs'); }
11
+ return settings;
12
+ }
13
+ async function body(response, maxBytes) {
14
+ if (!response.ok) { throw new Error(`Loki HTTP status ${response.status}`); }
15
+ if (!response.body) { throw new Error('Loki response body missing'); }
16
+ const chunks = [];
17
+ let bytes = 0;
18
+ for await (const chunk of response.body) {
19
+ bytes += chunk.byteLength;
20
+ if (bytes > maxBytes) { throw new Error('Loki response capacity exceeded'); }
21
+ chunks.push(chunk);
22
+ }
23
+ return JSON.parse(Buffer.concat(chunks, bytes).toString());
24
+ }
25
+ function entries(data, limit) {
26
+ if (data.status !== 'success' || !Array.isArray(data.data?.result) || data.data.resultType !== 'streams') {
27
+ throw new Error('Invalid Loki stream response');
28
+ }
29
+ const rows = data.data.result.flatMap((stream) => {
30
+ if (!Array.isArray(stream.values)) { throw new Error('Invalid Loki stream values'); }
31
+ return stream.values;
32
+ });
33
+ // A full page may hide older/equal-timestamp rows. Do not deliver or advance a partial window.
34
+ if (rows.length >= limit) { throw new Error('Loki query page saturated; history incomplete, cursor unchanged'); }
35
+ return rows.map(([ts, line]) => {
36
+ if (typeof ts !== 'string' || !/^\d+$/u.test(ts) || typeof line !== 'string') { throw new Error('Invalid Loki log entry'); }
37
+ return { ts: Number(ts) / 1000000, line };
38
+ });
39
+ }
40
+ function requestUrl(url, query, range, limit) {
41
+ const params = new URLSearchParams({ query, start: (range.since * 1000000).toString(),
42
+ end: (range.until * 1000000).toString(), limit: String(limit), direction: 'forward' });
43
+ return `${url}/loki/api/v1/query_range?${params}`;
44
+ }
45
+ async function request(url, settings, signal) {
46
+ const controller = new AbortController();
47
+ function abort() { controller.abort(signal.reason); }
48
+ if (signal?.aborted) { abort(); }
49
+ signal?.addEventListener('abort', abort, { once: true });
50
+ const timer = setTimeout(() => { controller.abort(new Error('Loki request deadline exceeded')); }, settings.requestTimeoutMs);
51
+ try {
52
+ const response = await globalThis.fetch(url, { signal: controller.signal });
53
+ return entries(await body(response, settings.maxResponseBytes), settings.pageLimit);
54
+ } catch (error) {
55
+ controller.abort(error);
56
+ try { settings.onError?.(error); } catch {
57
+ // Diagnostics cannot suppress the failed read or change its cursor.
58
+ }
59
+ throw error;
60
+ } finally {
61
+ clearTimeout(timer);
62
+ signal?.removeEventListener('abort', abort);
63
+ }
64
+ }
65
+ /**
66
+ * Bounded Loki polling. Saturated pages fail without advancing the in-memory cursor.
67
+ * Windows lag wall-clock time by queryLagMs (default 1s) to allow ingestion visibility.
68
+ * Entries becoming visible later than this configured lag may still be missed.
69
+ * This live polling source does not expose portable history checkpoints or pagination.
70
+ *
71
+ * @param {string} url - Loki base URL
72
+ * @param {string} query - LogQL expression
73
+ * @param {number} interval - Polling interval in seconds
74
+ * @param {object} collector - Collector with accept()
75
+ * @param {object} clk - Clock with millis()
76
+ * @param {object} options - requestTimeoutMs, maxResponseBytes, pageLimit, queryLagMs, onError
77
+ * @returns {object} Source with async start/stop
78
+ */
79
+ // Sixth optional argument preserves the established five-argument source API.
80
+ // eslint-disable-next-line max-params
81
+ export default function lokiSource(url, query, interval, collector, clk, options = {}) {
82
+ if (typeof url !== 'string' || url.length === 0) { throw new Error('URL must be a non-empty string'); }
83
+ if (typeof query !== 'string' || query.length === 0) { throw new Error('Query must be a non-empty string'); }
84
+ const settings = config(options);
85
+ return pollingSource((since, until, { signal } = {}) => {
86
+ return request(requestUrl(url, query, { since: since - settings.queryLagMs, until: until - settings.queryLagMs }, settings.pageLimit), settings, signal);
87
+ }, interval, collector, clk);
88
+ }