@yarkivaev/scada 2.3.116 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -0
- package/charts/scada/templates/processes.yaml +22 -0
- package/charts/scada/values.schema.json +12 -0
- package/db/migrations/V0009__collector_task_state.sql +6 -0
- package/package.json +11 -5
- package/src/application/deploy/readDocument.js +4 -1
- package/src/application/plantServer.js +1 -1
- package/src/application/runtime/additionalAdapters.js +139 -0
- package/src/application/runtime/assignment.js +30 -0
- package/src/application/runtime/budgetedSink.js +17 -0
- package/src/application/runtime/collectorSignals.js +14 -0
- package/src/application/runtime/compositeLifecycle.js +49 -0
- package/src/application/runtime/createRegistry.js +20 -0
- package/src/application/runtime/deadline.js +8 -0
- package/src/application/runtime/index.js +10 -0
- package/src/application/runtime/manifest.js +38 -0
- package/src/application/runtime/pipelineAdapters.js +60 -0
- package/src/application/runtime/resolveSecrets.js +12 -0
- package/src/application/runtime/resourceClaims.js +21 -0
- package/src/application/runtime/runCollector.js +50 -0
- package/src/application/runtime/runtime.js +86 -0
- package/src/application/runtime/runtimeHealth.js +29 -0
- package/src/application/runtime/runtimeServices.js +15 -0
- package/src/application/runtime/stateDocument.js +22 -0
- package/src/application/runtime/streamPorts.js +42 -0
- package/src/application/runtime/streamTask.js +105 -0
- package/src/application/runtime/taskBudget.js +31 -0
- package/src/application/runtime/taskContext.js +18 -0
- package/src/application/runtime/taskQueue.js +21 -0
- package/src/application/runtime/taskSlot.js +46 -0
- package/src/application/runtime/taskState.js +73 -0
- package/src/application/siteCollection.js +14 -0
- package/src/application/siteServer.js +7 -6
- package/src/application/siteTelemetry.js +1 -1
- package/src/application/supervisorSink.js +9 -8
- package/src/application/sync/siteSyncBind.js +1 -1
- package/src/application/sync/siteSyncTargets.js +1 -1
- package/src/bin/collector.js +7 -0
- package/src/bin/modbus-to-mqtt.js +11 -9
- package/src/infrastructure/ingest/activity/activityTracking.js +15 -21
- package/src/infrastructure/ingest/activity/activityTransformer.js +16 -15
- package/src/infrastructure/ingest/durableConsume.js +2 -2
- package/src/infrastructure/ingest/ingestBatch.js +13 -0
- package/src/infrastructure/ingest/modbus/mx210Tcp.js +4 -4
- package/src/infrastructure/ingest/modbus/silentStreams.js +33 -8
- package/src/infrastructure/ingest/mqtt/metricsTransformer.js +4 -5
- package/src/infrastructure/ingest/mqtt/modbusMqtt.js +17 -19
- package/src/infrastructure/ingest/mqtt/modbusPublish.js +4 -0
- package/src/infrastructure/ingest/mqtt/mqttMetrics.js +17 -21
- package/src/infrastructure/ingest/mqtt/registerPublish.js +3 -3
- package/src/infrastructure/ingest/pipelines/alertPipeline.js +7 -6
- package/src/infrastructure/ingest/pipelines/decisionPipeline.js +7 -6
- package/src/infrastructure/ingest/pipelines/segmentPipeline.js +8 -17
- package/src/infrastructure/ingest/pipelines/stompOptions.js +7 -0
- package/src/infrastructure/ingest/telemetry/amqpMetricsIngest.js +10 -11
- package/src/infrastructure/ingest/telemetry/amqpMqttRelay.js +10 -6
- package/src/infrastructure/ingest/telemetry/foldingSink.js +5 -2
- package/src/infrastructure/messaging/stomp/segmentRetags.js +1 -1
- package/src/infrastructure/messaging/stomp/userDecisions.js +1 -1
- package/src/infrastructure/opcua/certificates.js +11 -0
- package/src/infrastructure/opcua/delivery.js +38 -0
- package/src/infrastructure/opcua/endpoint.js +12 -2
- package/src/infrastructure/opcua/session.js +20 -10
- package/src/infrastructure/opcua/source.js +6 -3
- package/src/infrastructure/persistence/runtime/taskStatePg.js +24 -0
- package/src/infrastructure/streams/README.md +291 -0
- package/src/infrastructure/streams/abortable.js +14 -0
- package/src/infrastructure/streams/amqpSink/publication.js +46 -0
- package/src/infrastructure/streams/amqpSink/session.js +105 -0
- package/src/infrastructure/streams/amqpSink.js +47 -0
- package/src/infrastructure/streams/amqpSource/session.js +127 -0
- package/src/infrastructure/streams/amqpSource.js +97 -0
- package/src/infrastructure/streams/brokerClose.js +19 -0
- package/src/infrastructure/streams/budgetAccept.js +6 -0
- package/src/infrastructure/streams/clickhouseSink.js +48 -0
- package/src/infrastructure/streams/deadline.js +1 -0
- package/src/infrastructure/streams/follow.js +41 -0
- package/src/infrastructure/streams/index.js +15 -0
- package/src/infrastructure/streams/lokiSource.js +88 -0
- package/src/infrastructure/streams/modbusConnection.js +173 -0
- package/src/infrastructure/streams/modbusRtuBusSource.js +124 -0
- package/src/infrastructure/streams/modbusSource.js +185 -0
- package/src/infrastructure/streams/mqttSink/session.js +51 -0
- package/src/infrastructure/streams/mqttSink.js +54 -0
- package/src/infrastructure/streams/mqttSource.js +66 -0
- package/src/infrastructure/streams/pollingSource.js +1 -0
- package/src/infrastructure/streams/positive.js +5 -0
- package/src/infrastructure/streams/postgresSink.js +124 -0
- package/src/infrastructure/streams/serialReceive.js +80 -0
- package/src/infrastructure/streams/serialSource.js +87 -0
- package/src/infrastructure/streams/sqliteSink.js +55 -0
- package/src/infrastructure/streams/stompBody.js +46 -0
- package/src/infrastructure/streams/stompSend.js +89 -0
- package/src/infrastructure/streams/stompSource.js +201 -0
- package/src/infrastructure/sync/operationSyncIngest.js +1 -1
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import pg from 'pg';
|
|
2
|
+
import deadline from './deadline.js';
|
|
3
|
+
import positive from './positive.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Removes duplicate records by conflict key within a batch.
|
|
7
|
+
*
|
|
8
|
+
* PostgreSQL rejects INSERT batches where ON CONFLICT would update
|
|
9
|
+
* the same row twice. This keeps the last occurrence for each
|
|
10
|
+
* unique combination of conflict column values.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* deduplicate(
|
|
14
|
+
* [{ machine: 'a', start: 1, name: 'pending' },
|
|
15
|
+
* { machine: 'a', start: 1, name: 'completed' }],
|
|
16
|
+
* ['machine', 'start']
|
|
17
|
+
* );
|
|
18
|
+
* // => [{ machine: 'a', start: 1, name: 'completed' }]
|
|
19
|
+
*
|
|
20
|
+
* @param {Array} records - Array of record objects
|
|
21
|
+
* @param {Array<string>} keys - Conflict column names
|
|
22
|
+
* @returns {Array} Deduplicated records
|
|
23
|
+
*/
|
|
24
|
+
export function deduplicate(records, keys) {
|
|
25
|
+
if (keys.length === 0) {return records;}
|
|
26
|
+
const seen = new Map();
|
|
27
|
+
for (const record of records) {
|
|
28
|
+
const key = keys.map(k => {return record[k]}).join('\0');
|
|
29
|
+
seen.set(key, record);
|
|
30
|
+
}
|
|
31
|
+
return Array.from(seen.values());
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Builds the ON CONFLICT SQL suffix from options.
|
|
36
|
+
*
|
|
37
|
+
* @param {object} options - Sink options with conflict, update, and optional updateWhere
|
|
38
|
+
* @returns {string} SQL suffix or empty string
|
|
39
|
+
*/
|
|
40
|
+
function buildSuffix(options) {
|
|
41
|
+
if (!Array.isArray(options.conflict) || options.conflict.length === 0) {return '';}
|
|
42
|
+
const cols = options.conflict.join(', ');
|
|
43
|
+
if (Array.isArray(options.update) && options.update.length > 0) {
|
|
44
|
+
const sets = options.update.map((col) => {return `${col} = EXCLUDED.${col}`}).join(', ');
|
|
45
|
+
const guard = typeof options.updateWhere === 'string' && options.updateWhere.length > 0
|
|
46
|
+
? ` WHERE ${options.updateWhere}`
|
|
47
|
+
: '';
|
|
48
|
+
return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}${guard}`;
|
|
49
|
+
}
|
|
50
|
+
return ` ON CONFLICT (${cols}) DO NOTHING`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* PostgreSQL sink for batch record insertion.
|
|
55
|
+
*
|
|
56
|
+
* Creates a pg.Pool internally unless `options.pool` is supplied.
|
|
57
|
+
* Supports optional conflict resolution via ON CONFLICT DO NOTHING or
|
|
58
|
+
* ON CONFLICT DO UPDATE SET. `write` accepts an optional queryable
|
|
59
|
+
* client so callers can run inserts inside an existing transaction.
|
|
60
|
+
*
|
|
61
|
+
* @example
|
|
62
|
+
* const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
|
|
63
|
+
* await sink.write([{ ts: Date.now(), value: 42 }]);
|
|
64
|
+
*
|
|
65
|
+
* @example
|
|
66
|
+
* const sink = postgresSink(url, 'segments', cols, { pool, conflict: ['machine', 'start_time'] });
|
|
67
|
+
* await sink.write([{ machine: 'a', name: 'on', start_time: '2024-01-01' }], client);
|
|
68
|
+
*
|
|
69
|
+
* @param {string} url - PostgreSQL URL (e.g., 'postgresql://localhost:5432/db')
|
|
70
|
+
* @param {string} table - Target table name
|
|
71
|
+
* @param {Array<string>} columns - Column names for insertion
|
|
72
|
+
* @param {object} [options] - Optional configuration
|
|
73
|
+
* @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
|
|
74
|
+
* @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
|
|
75
|
+
* @param {string} [options.updateWhere] - Optional WHERE guard on DO UPDATE
|
|
76
|
+
* @param {object} [options.pool] - Existing pg Pool to reuse
|
|
77
|
+
* @returns {object} Sink with write(records, client?) method
|
|
78
|
+
*/
|
|
79
|
+
export default function postgresSink(url, table, columns, options = {}) {
|
|
80
|
+
if (typeof url !== 'string' || url.length === 0) {
|
|
81
|
+
throw new Error('URL must be a non-empty string');
|
|
82
|
+
}
|
|
83
|
+
if (typeof table !== 'string' || table.length === 0) {
|
|
84
|
+
throw new Error('Table must be a non-empty string');
|
|
85
|
+
}
|
|
86
|
+
if (!Array.isArray(columns) || columns.length === 0) {
|
|
87
|
+
throw new Error('Columns must be a non-empty array');
|
|
88
|
+
}
|
|
89
|
+
positive(options.stopTimeoutMs ?? 5000, 'stopTimeoutMs');
|
|
90
|
+
const pool = options.pool || (options.createPool || ((config) => { return new pg.Pool(config); }))({ connectionString: url });
|
|
91
|
+
let stopping;
|
|
92
|
+
const suffix = buildSuffix(options);
|
|
93
|
+
const conflict = Array.isArray(options.conflict) ? options.conflict : [];
|
|
94
|
+
return {
|
|
95
|
+
/**
|
|
96
|
+
* Writes records to PostgreSQL table.
|
|
97
|
+
*
|
|
98
|
+
* @param {Array} records - Array of objects with keys matching columns
|
|
99
|
+
* @param {object} [client] - Optional pg client/pool with query()
|
|
100
|
+
* @returns {Promise} Promise resolving when insert completes
|
|
101
|
+
*/
|
|
102
|
+
write(records, client) {
|
|
103
|
+
if (stopping) { return Promise.reject(new Error('PostgreSQL sink stopped')); }
|
|
104
|
+
if (client?.signal?.aborted) { return Promise.reject(client.signal.reason || new Error('PostgreSQL write aborted')); }
|
|
105
|
+
const unique = deduplicate(records, conflict);
|
|
106
|
+
const placeholders = unique.map((record, i) => {
|
|
107
|
+
const offset = i * columns.length;
|
|
108
|
+
const row = columns.map((__, j) => {return `$${offset + j + 1}`}).join(', ');
|
|
109
|
+
return `(${row})`;
|
|
110
|
+
}).join(', ');
|
|
111
|
+
const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
|
|
112
|
+
const values = unique.flatMap((record) => {return columns.map((col) => {return record[col]})});
|
|
113
|
+
const runner = client && typeof client.query === 'function' ? client : pool;
|
|
114
|
+
return runner.query(query, values);
|
|
115
|
+
},
|
|
116
|
+
stop() {
|
|
117
|
+
stopping ??= deadline(Promise.resolve().then(() => {
|
|
118
|
+
if (!options.pool) { return pool.end(); }
|
|
119
|
+
return undefined;
|
|
120
|
+
}), options.stopTimeoutMs ?? 5000, 'PostgreSQL pool close');
|
|
121
|
+
return stopping;
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import deadline from './deadline.js';
|
|
2
|
+
|
|
3
|
+
export function report(options, error, device) {
|
|
4
|
+
try { options.onError?.(error, device); } catch {
|
|
5
|
+
// Diagnostics cannot prevent shutdown.
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
export function closePort(port, timeoutMs) {
|
|
9
|
+
if (port.isOpen === false) { return Promise.resolve(); }
|
|
10
|
+
return deadline(new Promise((resolve, reject) => {
|
|
11
|
+
port.close((error) => { if (error) { reject(error); } else { resolve(); } });
|
|
12
|
+
}), timeoutMs, 'Serial close', () => { port.destroy?.(); }).catch((error) => {
|
|
13
|
+
error.resourceUncertain = true;
|
|
14
|
+
throw error;
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
function reserve(ctx, size) {
|
|
18
|
+
try {
|
|
19
|
+
const release = ctx.options.budget?.reserve({ bytes: size, records: 1 });
|
|
20
|
+
ctx.port.pause?.();
|
|
21
|
+
return { release };
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
Object.assign(ctx.state, { fatal: error, active: false });
|
|
25
|
+
ctx.state.controller.abort(error);
|
|
26
|
+
ctx.fail(error);
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function accept(ctx, chunk, release) {
|
|
31
|
+
const accepted = Promise.resolve().then(() => {
|
|
32
|
+
if (ctx.state.active && !ctx.failed) { return ctx.collector.accept({ device: ctx.device, chunk }, { signal: ctx.state.controller.signal }); }
|
|
33
|
+
return undefined;
|
|
34
|
+
});
|
|
35
|
+
void accepted.finally(() => { release?.(); }).catch(() => { return undefined; });
|
|
36
|
+
return deadline(accepted, ctx.options.acceptTimeoutMs, 'Serial acceptance', (error) => {
|
|
37
|
+
ctx.state.fatal = error; ctx.state.active = false; ctx.state.controller.abort(error);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
function enqueue(ctx, chunk) {
|
|
41
|
+
if (!ctx.state.active) { return; }
|
|
42
|
+
const size = chunk.length ?? Buffer.byteLength(String(chunk));
|
|
43
|
+
if (ctx.count >= ctx.options.maxPending || ctx.bytes + size > ctx.options.maxPendingBytes) {
|
|
44
|
+
const error = new Error('Serial receive capacity exceeded; source retired');
|
|
45
|
+
Object.assign(ctx.state, { fatal: error, active: false });
|
|
46
|
+
ctx.state.controller.abort(error);
|
|
47
|
+
ctx.fail(error);
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const lease = reserve(ctx, size);
|
|
51
|
+
if (!lease) { return; }
|
|
52
|
+
Object.assign(ctx, { count: ctx.count + 1, bytes: ctx.bytes + size });
|
|
53
|
+
ctx.pending = ctx.pending.then(() => {
|
|
54
|
+
if (ctx.state.active && !ctx.failed) {
|
|
55
|
+
return accept(ctx, chunk, lease.release);
|
|
56
|
+
}
|
|
57
|
+
lease.release?.();
|
|
58
|
+
return undefined;
|
|
59
|
+
}).catch(ctx.fail).finally(() => {
|
|
60
|
+
ctx.count -= 1;
|
|
61
|
+
ctx.bytes -= size;
|
|
62
|
+
if (!ctx.count && ctx.state.active && !ctx.failed) { ctx.port.resume?.(); }
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
/** Bound data even if a custom driver ignores pause; pause/resume the native stream. */
|
|
66
|
+
export default async function receive(state, port, device, collector, options) {
|
|
67
|
+
const ctx = { state, port, device, collector, options, pending: Promise.resolve(), count: 0, bytes: 0, failed: false };
|
|
68
|
+
const closed = new Promise((resolve) => { state.wake = resolve; });
|
|
69
|
+
ctx.fail = (error) => { ctx.failed = true; report(options, error, device); state.wake?.(); };
|
|
70
|
+
function data(chunk) { enqueue(ctx, chunk); }
|
|
71
|
+
function close() { state.wake?.(); }
|
|
72
|
+
port.on('data', data).on('error', ctx.fail).on('close', close);
|
|
73
|
+
if (!state.active) { state.wake(); }
|
|
74
|
+
await closed;
|
|
75
|
+
port.off('data', data);
|
|
76
|
+
try {
|
|
77
|
+
await closePort(port, options.stopTimeoutMs);
|
|
78
|
+
await deadline(ctx.pending, options.stopTimeoutMs, 'Serial acceptance', (error) => { state.controller.abort(error); });
|
|
79
|
+
} finally { port.off('error', ctx.fail).off('close', close); }
|
|
80
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import positive from './positive.js';
|
|
2
|
+
|
|
3
|
+
import deadline from './deadline.js';
|
|
4
|
+
import receive, { closePort, report } from './serialReceive.js';
|
|
5
|
+
|
|
6
|
+
async function openPort(device) {
|
|
7
|
+
const { SerialPort } = await import('serialport');
|
|
8
|
+
const port = new SerialPort({ ...device, autoOpen: false });
|
|
9
|
+
await new Promise((resolve, reject) => { port.open((error) => { if (error) { reject(error); } else { resolve(); } }); });
|
|
10
|
+
return port;
|
|
11
|
+
}
|
|
12
|
+
function pause(state, milliseconds) {
|
|
13
|
+
return new Promise((resolve) => { state.wake = resolve; state.timer = setTimeout(resolve, milliseconds); })
|
|
14
|
+
.finally(() => { clearTimeout(state.timer); state.wake = null; });
|
|
15
|
+
}
|
|
16
|
+
async function run(state, device, collector, options) {
|
|
17
|
+
while (state.active) {
|
|
18
|
+
try {
|
|
19
|
+
// Each port has an independent serialized open/close/retry loop.
|
|
20
|
+
const attempt = { timedOut: false };
|
|
21
|
+
const opening = Promise.resolve().then(() => { return options.openPort(device, { signal: state.controller.signal }); });
|
|
22
|
+
void opening.then((late) => {
|
|
23
|
+
if (attempt.timedOut) { return closePort(late, options.stopTimeoutMs); }
|
|
24
|
+
return undefined;
|
|
25
|
+
}).catch((error) => { report(options, error, device); });
|
|
26
|
+
// A timed-out open is uncertain; never open a second handle in this instance.
|
|
27
|
+
// eslint-disable-next-line no-await-in-loop
|
|
28
|
+
const port = await deadline(opening, options.openTimeoutMs, 'Serial open', (error) => { attempt.timedOut = true; state.controller.abort(error); });
|
|
29
|
+
// eslint-disable-next-line no-await-in-loop
|
|
30
|
+
await receive(state, port, device, collector, options);
|
|
31
|
+
} catch (error) {
|
|
32
|
+
if (error.resourceUncertain || error.code === 'OPERATION_TIMEOUT') { Object.assign(state, { fatal: error, active: false }); }
|
|
33
|
+
report(options, error, device);
|
|
34
|
+
}
|
|
35
|
+
if (state.active) {
|
|
36
|
+
// eslint-disable-next-line no-await-in-loop
|
|
37
|
+
await pause(state, options.retryMs);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function validate(devices, collector) {
|
|
42
|
+
if (!Array.isArray(devices)) { throw new TypeError('devices must be an array'); }
|
|
43
|
+
if (!collector || typeof collector.accept !== 'function') { throw new TypeError('Collector must have an accept() method'); }
|
|
44
|
+
const paths = new Set();
|
|
45
|
+
devices.forEach((device) => {
|
|
46
|
+
if (!device || typeof device.path !== 'string' || !device.path.trim() || paths.has(device.path)) {
|
|
47
|
+
throw new TypeError('invalid or duplicate serial path');
|
|
48
|
+
}
|
|
49
|
+
paths.add(device.path);
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
/** Independent serial devices; chunks and device metadata pass through unchanged. */
|
|
53
|
+
export default function serialSource(devices, collector, options = {}) {
|
|
54
|
+
validate(devices, collector);
|
|
55
|
+
const settings = { retryMs: 1000, openTimeoutMs: 5000, acceptTimeoutMs: 10000, stopTimeoutMs: 5000, maxPending: 32, maxPendingBytes: 1048576,
|
|
56
|
+
...options, openPort: options.openPort ?? openPort };
|
|
57
|
+
['retryMs', 'acceptTimeoutMs', 'openTimeoutMs', 'stopTimeoutMs', 'maxPending', 'maxPendingBytes'].forEach((key) => { positive(settings[key], key); });
|
|
58
|
+
if (!Number.isSafeInteger(settings.maxPending) || !Number.isSafeInteger(settings.maxPendingBytes)) { throw new TypeError('Serial capacities must be safe integers'); }
|
|
59
|
+
const states = devices.map(() => { return { active: false, task: null, wake: null, timer: null }; });
|
|
60
|
+
let revision = 0;
|
|
61
|
+
let stopping = Promise.resolve();
|
|
62
|
+
return {
|
|
63
|
+
async start() {
|
|
64
|
+
revision += 1;
|
|
65
|
+
const requested = revision;
|
|
66
|
+
await stopping;
|
|
67
|
+
const failed = states.find((state) => { return state.fatal; });
|
|
68
|
+
if (failed) { throw failed.fatal; }
|
|
69
|
+
if (requested === revision) {
|
|
70
|
+
states.forEach((state, index) => {
|
|
71
|
+
if (!state.active) { state.active = true; state.controller = new AbortController(); state.task = run(state, devices[index], collector, settings); }
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
stop() {
|
|
76
|
+
revision += 1;
|
|
77
|
+
states.forEach((state) => { state.active = false; state.controller?.abort(new Error('Serial source stopped')); state.wake?.(); });
|
|
78
|
+
stopping = deadline(Promise.all(states.map((state) => { return state.task; })), settings.stopTimeoutMs, 'Serial stop', (error) => {
|
|
79
|
+
states.forEach((state) => { state.fatal = error; });
|
|
80
|
+
}).then(() => {
|
|
81
|
+
const failed = states.find((state) => { return state.fatal; });
|
|
82
|
+
if (failed) { throw failed.fatal; }
|
|
83
|
+
});
|
|
84
|
+
return stopping;
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* SQLite sink for batch record insertion.
|
|
5
|
+
*
|
|
6
|
+
* Opens a SQLite database in WAL mode and implements the Sink
|
|
7
|
+
* interface for use with batch collectors. Creates the target
|
|
8
|
+
* table with the given columns if it does not exist.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* const sink = sqliteSink('/tmp/metrics.db', 'metrics', ['topic', 'ts', 'value']);
|
|
12
|
+
* sink.write([{ topic: 'sensor/1', ts: 1700000000000, value: 42 }]);
|
|
13
|
+
*
|
|
14
|
+
* @param {string} path - Filesystem path to the SQLite database file
|
|
15
|
+
* @param {string} table - Target table name
|
|
16
|
+
* @param {Array<string>} columns - Column names for insertion
|
|
17
|
+
* @returns {object} Sink with write(records) and close() methods
|
|
18
|
+
*/
|
|
19
|
+
export default function sqliteSink(path, table, columns) {
|
|
20
|
+
if (typeof path !== 'string' || path.length === 0) {
|
|
21
|
+
throw new Error('Path must be a non-empty string');
|
|
22
|
+
}
|
|
23
|
+
if (typeof table !== 'string' || table.length === 0) {
|
|
24
|
+
throw new Error('Table must be a non-empty string');
|
|
25
|
+
}
|
|
26
|
+
if (!Array.isArray(columns) || columns.length === 0) {
|
|
27
|
+
throw new Error('Columns must be a non-empty array');
|
|
28
|
+
}
|
|
29
|
+
const db = new Database(path);
|
|
30
|
+
db.pragma('journal_mode = WAL');
|
|
31
|
+
const placeholders = columns.map(() => {return '?'}).join(', ');
|
|
32
|
+
const sql = `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${placeholders})`;
|
|
33
|
+
const statement = db.prepare(sql);
|
|
34
|
+
const transaction = db.transaction((records) => {
|
|
35
|
+
for (const record of records) {
|
|
36
|
+
statement.run(...columns.map((col) => {return record[col]}));
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
return {
|
|
40
|
+
/**
|
|
41
|
+
* Writes records to SQLite table inside a transaction.
|
|
42
|
+
*
|
|
43
|
+
* @param {Array} records - Array of objects with keys matching columns
|
|
44
|
+
*/
|
|
45
|
+
write(records) {
|
|
46
|
+
transaction(records);
|
|
47
|
+
},
|
|
48
|
+
/**
|
|
49
|
+
* Closes the database connection.
|
|
50
|
+
*/
|
|
51
|
+
close() {
|
|
52
|
+
db.close();
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import positive from './positive.js';
|
|
2
|
+
import budgetAccept from './budgetAccept.js';
|
|
3
|
+
|
|
4
|
+
export function stompSettings(options, prefetch) {
|
|
5
|
+
const config = { acceptTimeoutMs: 10000, stopTimeoutMs: 5000, maxMessageBytes: 1048576, ...options };
|
|
6
|
+
config.maxPendingBytes ??= prefetch * config.maxMessageBytes;
|
|
7
|
+
['acceptTimeoutMs', 'stopTimeoutMs', 'maxMessageBytes', 'maxPendingBytes'].forEach((key) => { positive(config[key], key); });
|
|
8
|
+
if (!Number.isSafeInteger(config.maxMessageBytes) || !Number.isSafeInteger(config.maxPendingBytes)) {
|
|
9
|
+
throw new TypeError('STOMP byte capacities must be safe integers');
|
|
10
|
+
}
|
|
11
|
+
return config;
|
|
12
|
+
}
|
|
13
|
+
function overflow() { return new Error('STOMP payload capacity exceeded'); }
|
|
14
|
+
function legacyRead(message, maximum) {
|
|
15
|
+
return new Promise((resolve, reject) => {
|
|
16
|
+
message.readString('utf-8', (error, body) => {
|
|
17
|
+
if (error) { reject(error); }
|
|
18
|
+
else if (Buffer.byteLength(body) > maximum) { reject(overflow()); }
|
|
19
|
+
else { resolve(body.toString()); }
|
|
20
|
+
});
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
/** Native stompit frames are Readables: bound bytes before concatenating the body. */
|
|
24
|
+
export async function readStomp(message, maximum) {
|
|
25
|
+
if (!message[Symbol.asyncIterator]) { return legacyRead(message, maximum); }
|
|
26
|
+
const chunks = [];
|
|
27
|
+
let bytes = 0;
|
|
28
|
+
for await (const chunk of message) {
|
|
29
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
30
|
+
bytes += buffer.length;
|
|
31
|
+
if (bytes > maximum) { throw overflow(); }
|
|
32
|
+
chunks.push(buffer);
|
|
33
|
+
}
|
|
34
|
+
return Buffer.concat(chunks, bytes).toString('utf8');
|
|
35
|
+
}
|
|
36
|
+
/** Counters and global leases follow real acceptance, including after an uncertain timeout. */
|
|
37
|
+
export async function acceptStomp(ctx, body, settle) {
|
|
38
|
+
if (!ctx.lifecycle.active) { throw new Error('STOMP source stopped'); }
|
|
39
|
+
const bytes = Buffer.byteLength(body);
|
|
40
|
+
if (ctx.lifecycle.bytes + bytes > ctx.lifecycle.maxPendingBytes) { throw overflow(); }
|
|
41
|
+
ctx.lifecycle.bytes += bytes;
|
|
42
|
+
try {
|
|
43
|
+
return await budgetAccept(ctx.collector, { destination: ctx.destination, payload: body, settle }, bytes,
|
|
44
|
+
ctx.lifecycle, { signal: ctx.lifecycle.controller.signal });
|
|
45
|
+
} finally { ctx.lifecycle.bytes -= bytes; }
|
|
46
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import stompit from 'stompit';
|
|
2
|
+
import deadline from './deadline.js';
|
|
3
|
+
import positive from './positive.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Builds STOMP connect options from a URL.
|
|
7
|
+
*
|
|
8
|
+
* @param {URL} parsed - Parsed URL object
|
|
9
|
+
* @param {object} options - Connection options
|
|
10
|
+
* @returns {object} stompit connect parameters
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* STOMP SEND headers that persist the body on a durable RabbitMQ queue.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} destination - STOMP destination
|
|
16
|
+
* @returns {object} headers including persistent true
|
|
17
|
+
*/
|
|
18
|
+
export function stompHeaders(destination) {
|
|
19
|
+
return {
|
|
20
|
+
destination,
|
|
21
|
+
'content-type': 'application/json',
|
|
22
|
+
persistent: 'true'
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function connectParams(parsed, options) {
|
|
27
|
+
return {
|
|
28
|
+
host: parsed.hostname,
|
|
29
|
+
port: parseInt(parsed.port, 10) || 61613,
|
|
30
|
+
connectHeaders: {
|
|
31
|
+
host: options.host || parsed.hostname,
|
|
32
|
+
login: options.login || '',
|
|
33
|
+
passcode: options.passcode || ''
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* One-shot STOMP publish of a JSON payload to a destination.
|
|
40
|
+
*
|
|
41
|
+
* Connects, sends one MESSAGE frame, disconnects. For operator decisions
|
|
42
|
+
* and other low-volume outbound events.
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* await stompSend('stomp://rabbitmq:61613', '/exchange/scada.user_decisions',
|
|
46
|
+
* { machine: 'icht2', start: 1700000000, user: 'hmi', tags: ['heating'] },
|
|
47
|
+
* { login: 'guest', passcode: 'guest', host: '/' });
|
|
48
|
+
*
|
|
49
|
+
* @param {string} url - STOMP broker URL
|
|
50
|
+
* @param {string} destination - STOMP destination
|
|
51
|
+
* @param {object|string} payload - JSON-serializable body or raw string
|
|
52
|
+
* @param {object} [options] - STOMP connection options
|
|
53
|
+
* @param {string} [options.login] - Login
|
|
54
|
+
* @param {string} [options.passcode] - Passcode
|
|
55
|
+
* @param {string} [options.host] - Virtual host header
|
|
56
|
+
* @returns {Promise<void>} Resolves when the frame is sent and client disconnects
|
|
57
|
+
*/
|
|
58
|
+
export default function stompSend(url, destination, payload, options = {}) {
|
|
59
|
+
if (typeof url !== 'string' || url.length === 0) {
|
|
60
|
+
throw new Error('URL must be a non-empty string');
|
|
61
|
+
}
|
|
62
|
+
if (typeof destination !== 'string' || destination.length === 0) {
|
|
63
|
+
throw new Error('Destination must be a non-empty string');
|
|
64
|
+
}
|
|
65
|
+
const parsed = new URL(url);
|
|
66
|
+
const params = connectParams(parsed, options);
|
|
67
|
+
const body = typeof payload === 'string' ? payload : JSON.stringify(payload);
|
|
68
|
+
const timeoutMs = options.timeoutMs ?? 10000;
|
|
69
|
+
positive(timeoutMs, 'timeoutMs');
|
|
70
|
+
const state = { active: true, client: null, connecting: null };
|
|
71
|
+
const sent = new Promise((resolve, reject) => {
|
|
72
|
+
state.connecting = (options.connect || stompit.connect)({ ...params, timeout: timeoutMs }, (error, client) => {
|
|
73
|
+
if (error) { reject(error); return; }
|
|
74
|
+
state.client = client;
|
|
75
|
+
if (!state.active) { client.destroy(); return; }
|
|
76
|
+
client.on('error', reject);
|
|
77
|
+
try {
|
|
78
|
+
const frame = client.send(stompHeaders(destination), { onReceipt() {
|
|
79
|
+
if (state.active) { client.disconnect((failure) => { if (failure) { reject(failure); } else { resolve(); } }); }
|
|
80
|
+
} });
|
|
81
|
+
frame.end(body);
|
|
82
|
+
} catch (failure) { client.destroy(); reject(failure); }
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
return deadline(sent, timeoutMs, 'STOMP receipt/disconnect', () => {
|
|
86
|
+
state.active = false;
|
|
87
|
+
(state.client || state.connecting)?.destroy();
|
|
88
|
+
}).finally(() => { state.active = false; });
|
|
89
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import stompit from 'stompit';
|
|
2
|
+
import follow from './follow.js';
|
|
3
|
+
import deadline from './deadline.js';
|
|
4
|
+
import { acceptStomp, readStomp, stompSettings } from './stompBody.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Idle state for STOMP source.
|
|
8
|
+
*
|
|
9
|
+
* @returns {object} State with subscribed() returning false
|
|
10
|
+
*/
|
|
11
|
+
function idle() {
|
|
12
|
+
return {
|
|
13
|
+
subscribed() {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Subscribed state for STOMP source.
|
|
21
|
+
*
|
|
22
|
+
* @param {object} channel - STOMP channel
|
|
23
|
+
* @returns {object} State with subscribed() returning true
|
|
24
|
+
*/
|
|
25
|
+
function subscribed(channel, delivery) {
|
|
26
|
+
return {
|
|
27
|
+
subscribed() {
|
|
28
|
+
return true;
|
|
29
|
+
},
|
|
30
|
+
channel() {
|
|
31
|
+
return channel;
|
|
32
|
+
},
|
|
33
|
+
delivery
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Builds STOMP server configuration from a URL.
|
|
39
|
+
*
|
|
40
|
+
* @param {URL} parsed - Parsed URL object
|
|
41
|
+
* @param {object} options - Connection options
|
|
42
|
+
* @returns {object} Server config for ConnectFailover
|
|
43
|
+
*/
|
|
44
|
+
function serverConfig(parsed, options) {
|
|
45
|
+
return {
|
|
46
|
+
host: parsed.hostname,
|
|
47
|
+
port: parseInt(parsed.port, 10) || 61613,
|
|
48
|
+
connectHeaders: {
|
|
49
|
+
host: options.host || parsed.hostname,
|
|
50
|
+
login: options.login || '',
|
|
51
|
+
passcode: options.passcode || ''
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Builds settle helpers for one STOMP frame.
|
|
58
|
+
*
|
|
59
|
+
* @param {object} channel - STOMP channel
|
|
60
|
+
* @param {object} message - STOMP frame
|
|
61
|
+
* @returns {{ ack: Function, nack: Function }}
|
|
62
|
+
*/
|
|
63
|
+
function settleOf(channel, message, lifecycle) {
|
|
64
|
+
let settled = false;
|
|
65
|
+
return {
|
|
66
|
+
ack() {
|
|
67
|
+
if (lifecycle.active && !settled) { channel.ack(message); settled = true; }
|
|
68
|
+
},
|
|
69
|
+
nack(requeue = true) {
|
|
70
|
+
if (lifecycle.active && !settled) { channel.nack(message, { requeue: String(requeue) }); settled = true; }
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Delivers one STOMP body to the collector and settles the frame.
|
|
77
|
+
*
|
|
78
|
+
* @param {object} ctx - delivery context with channel/message/destination/collector/manualAck
|
|
79
|
+
* @param {string} body - UTF-8 payload
|
|
80
|
+
* @returns {Promise<void>}
|
|
81
|
+
*/
|
|
82
|
+
async function deliver(ctx, body) {
|
|
83
|
+
const settle = settleOf(ctx.channel, ctx.message, ctx.lifecycle);
|
|
84
|
+
try {
|
|
85
|
+
await deadline(Promise.resolve().then(() => {
|
|
86
|
+
return acceptStomp(ctx, body, settle);
|
|
87
|
+
}), ctx.lifecycle.acceptTimeoutMs, 'STOMP acceptance', (error) => {
|
|
88
|
+
ctx.lifecycle.fatal = error; ctx.lifecycle.active = false; ctx.lifecycle.controller.abort(error); ctx.channel.close();
|
|
89
|
+
});
|
|
90
|
+
if (!ctx.manualAck) {
|
|
91
|
+
settle.ack();
|
|
92
|
+
}
|
|
93
|
+
} catch (error) {
|
|
94
|
+
if (!ctx.manualAck) {
|
|
95
|
+
settle.nack();
|
|
96
|
+
}
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Subscribes a channel and routes frames to the collector.
|
|
103
|
+
* Arrivals stay in subscribe order through follow, even when persist overlaps.
|
|
104
|
+
*
|
|
105
|
+
* @param {object} channel - STOMP channel
|
|
106
|
+
* @param {string} destination - queue destination
|
|
107
|
+
* @param {object} collector - message collector
|
|
108
|
+
* @param {object} delivery - serial, manualAck and prefetch settings
|
|
109
|
+
* @returns {undefined}
|
|
110
|
+
*/
|
|
111
|
+
function bind(channel, destination, collector, delivery) {
|
|
112
|
+
const { serial, manualAck, prefetch, lifecycle } = delivery;
|
|
113
|
+
let pending = Promise.resolve();
|
|
114
|
+
channel.subscribe({ destination, ack: 'client-individual', 'prefetch-count': String(prefetch) }, (err, message) => {
|
|
115
|
+
if (err) {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const ctx = { channel, message, destination, collector, manualAck, lifecycle };
|
|
119
|
+
pending = follow(pending, () => {
|
|
120
|
+
if (!lifecycle.active) { return Promise.reject(new Error('STOMP source stopped')); }
|
|
121
|
+
return deadline(readStomp(message, lifecycle.maxMessageBytes), lifecycle.acceptTimeoutMs, 'STOMP read').catch((error) => {
|
|
122
|
+
lifecycle.fatal = error; lifecycle.active = false; channel.close(); throw error;
|
|
123
|
+
});
|
|
124
|
+
}, (body) => {
|
|
125
|
+
const task = deliver(ctx, body);
|
|
126
|
+
lifecycle.pending.add(task);
|
|
127
|
+
return task.finally(() => { lifecycle.pending.delete(task); });
|
|
128
|
+
}, serial);
|
|
129
|
+
lifecycle.chain = pending;
|
|
130
|
+
void pending.catch((error) => {
|
|
131
|
+
try { lifecycle.onError?.(error); } catch {
|
|
132
|
+
// Reporting cannot reject a transport callback.
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* STOMP subscription source for streaming messages to a collector.
|
|
140
|
+
*
|
|
141
|
+
* Messages are `{destination, payload, settle}`. Default ack-after-accept;
|
|
142
|
+
* with `manualAck` the collector owns settle. Arrivals enqueue in
|
|
143
|
+
* subscribe order. With `serial: false` persist may overlap after enqueue.
|
|
144
|
+
* A finite prefetch (default 32) bounds unacknowledged broker deliveries.
|
|
145
|
+
*
|
|
146
|
+
* @param {string} url - STOMP broker URL
|
|
147
|
+
* @param {string} destination - STOMP destination
|
|
148
|
+
* @param {object} collector - Collector with accept()
|
|
149
|
+
* @param {object} [options] - Connection and delivery options
|
|
150
|
+
* @param {number} [options.prefetch=32] - Positive integer delivery window
|
|
151
|
+
* @returns {object} Source with start() and stop() methods
|
|
152
|
+
*/
|
|
153
|
+
export default function stompSource(url, destination, collector, options = {}) {
|
|
154
|
+
if (typeof url !== 'string' || url.length === 0) {
|
|
155
|
+
throw new Error('URL must be a non-empty string');
|
|
156
|
+
}
|
|
157
|
+
if (typeof destination !== 'string' || destination.length === 0) {
|
|
158
|
+
throw new Error('Destination must be a non-empty string');
|
|
159
|
+
}
|
|
160
|
+
if (!collector || typeof collector.accept !== 'function') {
|
|
161
|
+
throw new Error('Collector must have an accept() method');
|
|
162
|
+
}
|
|
163
|
+
const prefetch = options.prefetch ?? 32;
|
|
164
|
+
if (!Number.isSafeInteger(prefetch) || prefetch < 1) {
|
|
165
|
+
throw new Error('prefetch must be a positive safe integer');
|
|
166
|
+
}
|
|
167
|
+
const settings = stompSettings(options, prefetch);
|
|
168
|
+
const parsed = new URL(url);
|
|
169
|
+
const serial = options.serial !== false;
|
|
170
|
+
const manualAck = options.manualAck === true;
|
|
171
|
+
let state = idle();
|
|
172
|
+
return {
|
|
173
|
+
start() {
|
|
174
|
+
if (state.subscribed()) {
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
const failover = new stompit.ConnectFailover([serverConfig(parsed, options)], {
|
|
178
|
+
initialReconnectDelay: 10,
|
|
179
|
+
maxReconnectDelay: 30000,
|
|
180
|
+
maxReconnects: -1
|
|
181
|
+
});
|
|
182
|
+
const channel = new stompit.Channel(failover);
|
|
183
|
+
const lifecycle = { ...settings, active: true, controller: new AbortController(), pending: new Set(), bytes: 0 };
|
|
184
|
+
bind(channel, destination, collector, { serial, manualAck, prefetch, lifecycle });
|
|
185
|
+
state = subscribed(channel, lifecycle);
|
|
186
|
+
},
|
|
187
|
+
stop() {
|
|
188
|
+
if (!state.subscribed()) {
|
|
189
|
+
return Promise.resolve();
|
|
190
|
+
}
|
|
191
|
+
const stopped = state;
|
|
192
|
+
stopped.delivery.active = false;
|
|
193
|
+
stopped.delivery.controller.abort(new Error('STOMP source stopped'));
|
|
194
|
+
stopped.channel().close();
|
|
195
|
+
return deadline(Promise.allSettled([stopped.delivery.chain, ...stopped.delivery.pending]), options.stopTimeoutMs ?? 5000, 'STOMP stop').then(() => {
|
|
196
|
+
if (stopped.delivery.fatal) { throw stopped.delivery.fatal; }
|
|
197
|
+
if (state === stopped) { state = idle(); }
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
}
|