@yarkivaev/scada 2.3.113 → 2.3.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/charts/scada/Chart.yaml +2 -2
- package/charts/scada/templates/_helpers.tpl +12 -4
- package/charts/scada/templates/clickhouse.yaml +3 -3
- package/charts/scada/templates/collector.yaml +2 -2
- package/charts/scada/templates/namespace.yaml +6 -0
- package/charts/scada/templates/postgres.yaml +3 -3
- package/charts/scada/templates/processes.yaml +88 -0
- package/charts/scada/templates/rabbitmq-config.yaml +91 -0
- package/charts/scada/templates/rabbitmq.yaml +54 -107
- package/charts/scada/templates/site-server.yaml +5 -3
- package/charts/scada/templates/supervisor-sink.yaml +8 -2
- package/charts/scada/values.schema.json +22 -0
- package/charts/scada/values.yaml +22 -0
- package/index.js +3 -0
- package/package.json +5 -3
- package/src/application/deploy/collectorProcesses.js +42 -0
- package/src/application/deploy/expandFlows.js +21 -10
- package/src/application/deploy/publishedStreams.js +30 -0
- package/src/application/deploy/releaseEdges.js +13 -16
- package/src/application/deploy/releaseNodes.js +13 -0
- package/src/application/deploy/releaseValues.js +5 -15
- package/src/application/deploy/releases.js +3 -1
- package/src/application/deploy/subscriptions.js +20 -0
- package/src/application/supervisorSink.js +6 -0
- package/src/domain/segment/segmentCoalesce.js +26 -10
- package/src/infrastructure/deploy/executeRelease.js +60 -0
- package/src/infrastructure/ingest/codecs/segmentCodec.js +10 -3
- package/src/infrastructure/ingest/pipelines/segmentPipeline.js +3 -1
- package/src/infrastructure/messaging/rabbitmq/definitions.js +24 -0
- package/src/infrastructure/messaging/rabbitmq/reconcile.js +104 -0
package/charts/scada/values.yaml
CHANGED
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
role: edge
|
|
2
|
+
# Override the resource name prefix when adopting an existing installation.
|
|
3
|
+
prefix: ""
|
|
4
|
+
createNamespace: false
|
|
5
|
+
site:
|
|
6
|
+
enabled: true
|
|
7
|
+
sink:
|
|
8
|
+
enabled: true
|
|
9
|
+
# Labels for workloads that must stay outside immutable selectors.
|
|
10
|
+
workloadLabels: {}
|
|
11
|
+
# Named deployments with command, env, optional resources and hostDevices.
|
|
12
|
+
processes: []
|
|
2
13
|
image: ""
|
|
3
14
|
command: []
|
|
4
15
|
pullSecret: ""
|
|
@@ -21,6 +32,17 @@ postgres:
|
|
|
21
32
|
data: /var/lib/postgresql/data/pgdata
|
|
22
33
|
rabbitmq:
|
|
23
34
|
enabled: true
|
|
35
|
+
deadLetter: null
|
|
36
|
+
# A named standalone PVC replaces the default volumeClaimTemplates when set.
|
|
37
|
+
claimName: ""
|
|
38
|
+
configName: ""
|
|
39
|
+
rolloutOnConfigChange: false
|
|
40
|
+
definitions: null
|
|
41
|
+
plugins: [rabbitmq_stomp, rabbitmq_mqtt, rabbitmq_management]
|
|
42
|
+
extraConfig: {}
|
|
43
|
+
resources: {}
|
|
44
|
+
readinessProbe: {}
|
|
45
|
+
livenessProbe: {}
|
|
24
46
|
image: rabbitmq:3.13-management
|
|
25
47
|
storage: 5Gi
|
|
26
48
|
storageClass: ""
|
package/index.js
CHANGED
|
@@ -96,9 +96,12 @@ export { default as analyticsMcp } from './src/application/mcp/analyticsMcp.js';
|
|
|
96
96
|
export { default as expand } from './src/application/deploy/expand.js';
|
|
97
97
|
export { default as readDocument } from './src/application/deploy/readDocument.js';
|
|
98
98
|
export { default as mergeDocuments } from './src/application/deploy/mergeDocuments.js';
|
|
99
|
+
export { default as subscriptions } from './src/application/deploy/subscriptions.js';
|
|
99
100
|
export { default as releases } from './src/application/deploy/releases.js';
|
|
100
101
|
export { default as paths } from './src/application/deploy/paths.js';
|
|
101
102
|
export { default as appReleases } from './src/application/deploy/appReleases.js';
|
|
102
103
|
export { default as withCollectors } from './src/application/deploy/withCollectors.js';
|
|
103
104
|
export { default as pointPaths } from './src/application/deploy/pointPaths.js';
|
|
104
105
|
export { clusterOf, serverFor } from './src/application/deploy/releaseNodes.js';
|
|
106
|
+
export { default as executeRelease } from './src/infrastructure/deploy/executeRelease.js';
|
|
107
|
+
export { default as collectorProcesses } from './src/application/deploy/collectorProcesses.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yarkivaev/scada",
|
|
3
|
-
"version": "2.3.
|
|
3
|
+
"version": "2.3.115",
|
|
4
4
|
"description": "SCADA domain objects, state persistence, and plant monitoring",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -13,7 +13,9 @@
|
|
|
13
13
|
"./client": "./src/infrastructure/client/index.js",
|
|
14
14
|
"./stateDataFake": "./src/infrastructure/persistence/stateDataFromMemory.js",
|
|
15
15
|
"./startTestEdgeApi": "./src/infrastructure/http/edge/startTestEdgeApi.js",
|
|
16
|
-
"./package.json": "./package.json"
|
|
16
|
+
"./package.json": "./package.json",
|
|
17
|
+
"./rabbitmq/definitions": "./src/infrastructure/messaging/rabbitmq/definitions.js",
|
|
18
|
+
"./rabbitmq/reconcile": "./src/infrastructure/messaging/rabbitmq/reconcile.js"
|
|
17
19
|
},
|
|
18
20
|
"bin": {
|
|
19
21
|
"modbus-to-mqtt": "./src/bin/modbus-to-mqtt.js",
|
|
@@ -31,7 +33,7 @@
|
|
|
31
33
|
"@clickhouse/client": "^1.0.0",
|
|
32
34
|
"@modelcontextprotocol/sdk": "^1.24.3",
|
|
33
35
|
"@yarkivaev/simple-server": "1.0.2",
|
|
34
|
-
"@yarkivaev/source-to-sink": "1.3.
|
|
36
|
+
"@yarkivaev/source-to-sink": "1.3.10",
|
|
35
37
|
"amqplib": "^0.10.9",
|
|
36
38
|
"pg": "^8.0.0",
|
|
37
39
|
"stompit": "^1.0.0",
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
2
|
+
|
|
3
|
+
/** Group rows in first-seen order, keeping source order within each process. */
|
|
4
|
+
function grouped(machines, describe) {
|
|
5
|
+
const groups = new Map();
|
|
6
|
+
const claimed = new Set();
|
|
7
|
+
for (const row of machines) {
|
|
8
|
+
if (claimed.has(row.name)) { throw new Error(`duplicate source ${row.name}`); }
|
|
9
|
+
claimed.add(row.name);
|
|
10
|
+
const descriptor = describe(row);
|
|
11
|
+
if (typeof descriptor?.name !== 'string' || !descriptor.name.trim()) {
|
|
12
|
+
throw new Error(`unassigned source ${row.name}`);
|
|
13
|
+
}
|
|
14
|
+
const group = groups.get(descriptor.name) || { descriptor, rows: [] };
|
|
15
|
+
if (!isDeepStrictEqual(group.descriptor, descriptor)) {
|
|
16
|
+
throw new Error(`conflicting descriptions for process ${descriptor.name}`);
|
|
17
|
+
}
|
|
18
|
+
group.rows.push(row);
|
|
19
|
+
groups.set(descriptor.name, group);
|
|
20
|
+
}
|
|
21
|
+
return [...groups.values()];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* describe(row) returns a process descriptor with a name, command and optional
|
|
26
|
+
* env and resources. All sources describing the same process must agree.
|
|
27
|
+
* project(descriptor, rows) supplies adapter-specific env and optional hostDevices.
|
|
28
|
+
* A physical device may appear in many source rows but belongs to only one process.
|
|
29
|
+
*/
|
|
30
|
+
export default function collectorProcesses(machines, describe, project) {
|
|
31
|
+
const devices = new Set();
|
|
32
|
+
return grouped(machines, describe).map(({ descriptor, rows }) => {
|
|
33
|
+
const settings = project(descriptor, rows);
|
|
34
|
+
const hostDevices = [...new Set(settings.hostDevices || [])].sort();
|
|
35
|
+
hostDevices.forEach((device) => {
|
|
36
|
+
if (devices.has(device)) { throw new Error(`serial port or device ${device} is shared between processes`); }
|
|
37
|
+
devices.add(device);
|
|
38
|
+
});
|
|
39
|
+
return { name: descriptor.name, command: descriptor.command, env: { ...descriptor.env, ...settings.env },
|
|
40
|
+
hostDevices, resources: descriptor.resources };
|
|
41
|
+
});
|
|
42
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { assertLeaf, deviceFlows, expandObjects } from './expandDevices.js';
|
|
2
|
+
import { availableNames, expandApis, publishedNames } from './publishedStreams.js';
|
|
3
|
+
import { serviceOf } from './releaseNodes.js';
|
|
2
4
|
import { isAddress, picked } from './expandPattern.js';
|
|
3
5
|
|
|
4
6
|
/**
|
|
@@ -32,18 +34,17 @@ function chosen(pattern, pool, strict) {
|
|
|
32
34
|
* @example
|
|
33
35
|
* poolOf({ fanIn: [], fanOut: [] }, 'archive');
|
|
34
36
|
*/
|
|
35
|
-
function poolOf(source, receiverId) {
|
|
37
|
+
function poolOf(source, receiverId, api) {
|
|
38
|
+
const available = availableNames(source, api);
|
|
36
39
|
const aimed = source.fanOut.filter((flow) => {
|
|
37
40
|
return flow.to === receiverId;
|
|
38
41
|
}).map((flow) => {
|
|
39
42
|
return flow.name;
|
|
40
43
|
});
|
|
41
44
|
if (aimed.length > 0) {
|
|
42
|
-
return aimed;
|
|
45
|
+
return aimed.filter((name) => { return available.includes(name); });
|
|
43
46
|
}
|
|
44
|
-
return
|
|
45
|
-
return flow.name;
|
|
46
|
-
});
|
|
47
|
+
return available;
|
|
47
48
|
}
|
|
48
49
|
|
|
49
50
|
/**
|
|
@@ -60,9 +61,7 @@ function poolOf(source, receiverId) {
|
|
|
60
61
|
*/
|
|
61
62
|
function expandOut(server, ids, strict) {
|
|
62
63
|
const rows = [];
|
|
63
|
-
const streams = server
|
|
64
|
-
return flow.name;
|
|
65
|
-
});
|
|
64
|
+
const streams = availableNames(server);
|
|
66
65
|
server.patternsOut.forEach((flow) => {
|
|
67
66
|
const destinations = picked(flow.to, ids, 'server', true);
|
|
68
67
|
const names = picked(flow.name, streams, 'path', strict);
|
|
@@ -76,6 +75,18 @@ function expandOut(server, ids, strict) {
|
|
|
76
75
|
return rows;
|
|
77
76
|
}
|
|
78
77
|
|
|
78
|
+
/* A fixed object path can be held implicitly, but an API-only stream requires its service. */
|
|
79
|
+
function incomingNames(source, receiver, flow, strict) {
|
|
80
|
+
const api = serviceOf(source, flow.api)?.id;
|
|
81
|
+
const pool = poolOf(source, receiver, api);
|
|
82
|
+
const names = chosen(flow.name, pool, strict);
|
|
83
|
+
if (strict && api && publishedNames(source).includes(flow.name)
|
|
84
|
+
&& !availableNames(source, api).includes(flow.name)) {
|
|
85
|
+
throw new Error(`system document stream ${flow.name} is unavailable on api ${flow.api}`);
|
|
86
|
+
}
|
|
87
|
+
return names;
|
|
88
|
+
}
|
|
89
|
+
|
|
79
90
|
/**
|
|
80
91
|
* Expands fan-in whose source is a server.
|
|
81
92
|
*
|
|
@@ -101,7 +112,7 @@ function expandIn(server, drafts, strict) {
|
|
|
101
112
|
const source = drafts.find((item) => {
|
|
102
113
|
return item.id === sourceId;
|
|
103
114
|
});
|
|
104
|
-
|
|
115
|
+
incomingNames(source, server.id, flow, strict).forEach((path) => {
|
|
105
116
|
assertLeaf(server.objects, path);
|
|
106
117
|
const copy = { from: sourceId, name: path };
|
|
107
118
|
if (flow.api) {
|
|
@@ -144,7 +155,7 @@ function draftOne(server, paths, mappings) {
|
|
|
144
155
|
remote.push(flow);
|
|
145
156
|
});
|
|
146
157
|
return {
|
|
147
|
-
api: server.api,
|
|
158
|
+
api: expandApis(server.api, paths, objects),
|
|
148
159
|
fanIn: local,
|
|
149
160
|
id: server.id,
|
|
150
161
|
objects,
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { assertLeaf } from './expandDevices.js';
|
|
2
|
+
import { picked } from './expandPattern.js';
|
|
3
|
+
|
|
4
|
+
/** Resolve each API's published stream names, preserving its opaque contract metadata. */
|
|
5
|
+
export function expandApis(apis, paths, objects) {
|
|
6
|
+
return apis?.map((api) => {
|
|
7
|
+
if (!api.publishes) { return api; }
|
|
8
|
+
const publishes = api.publishes.flatMap((entry) => {
|
|
9
|
+
return picked(entry.name, paths, 'path', true).map((name) => {
|
|
10
|
+
assertLeaf(objects, name);
|
|
11
|
+
return { ...entry, name };
|
|
12
|
+
});
|
|
13
|
+
});
|
|
14
|
+
return { ...api, publishes };
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Streams made available by publishers on the selected service. */
|
|
19
|
+
export function publishedNames(server, api) {
|
|
20
|
+
return (server.api || []).filter((item) => {
|
|
21
|
+
return !api || item.id === api;
|
|
22
|
+
}).flatMap((item) => {
|
|
23
|
+
return (item.publishes || []).map(({ name }) => { return name; });
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Locally collected and forwarded streams stay available through the server's services. */
|
|
28
|
+
export function availableNames(source, api) {
|
|
29
|
+
return [...new Set([...source.fanIn.map(({ name }) => { return name; }), ...publishedNames(source, api)])];
|
|
30
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { publishedNames } from './publishedStreams.js';
|
|
1
2
|
import { isAddress } from './expandPattern.js';
|
|
2
3
|
import { asPath, objectPath, serverById } from './releaseNodes.js';
|
|
3
4
|
|
|
@@ -28,28 +29,22 @@ function sameObjects(left, right) {
|
|
|
28
29
|
* remember([], { from: 'cell', object: 'unit', side: 'out', to: 'archive' });
|
|
29
30
|
*/
|
|
30
31
|
function remember(edges, edge) {
|
|
31
|
-
|
|
32
|
+
let found = edges.find((item) => {
|
|
32
33
|
return item.from === edge.from && item.to === edge.to;
|
|
33
34
|
});
|
|
34
35
|
if (!found) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
from: edge.from,
|
|
38
|
-
objects: [edge.object],
|
|
39
|
-
sides: { [edge.side]: [edge.object] },
|
|
40
|
-
to: edge.to
|
|
41
|
-
});
|
|
42
|
-
return;
|
|
43
|
-
}
|
|
44
|
-
const prior = found.sides[edge.side] ? found.sides[edge.side].slice() : [];
|
|
45
|
-
if (!prior.includes(edge.object)) {
|
|
46
|
-
prior.push(edge.object);
|
|
36
|
+
found = { from: edge.from, to: edge.to, sides: {}, streams: new Map() };
|
|
37
|
+
edges.push(found);
|
|
47
38
|
}
|
|
48
|
-
found.sides[edge.side]
|
|
49
|
-
|
|
39
|
+
found.sides[edge.side] ||= [];
|
|
40
|
+
const objects = found.sides[edge.side];
|
|
41
|
+
if (!objects.includes(edge.object)) { objects.push(edge.object); }
|
|
42
|
+
const services = found.streams.get(edge.name) || new Set();
|
|
50
43
|
if (edge.api) {
|
|
44
|
+
services.add(edge.api);
|
|
51
45
|
found.api = edge.api;
|
|
52
46
|
}
|
|
47
|
+
found.streams.set(edge.name, services);
|
|
53
48
|
}
|
|
54
49
|
|
|
55
50
|
/**
|
|
@@ -67,6 +62,7 @@ export function edgesOf(document) {
|
|
|
67
62
|
const dest = serverById(document, flow.to);
|
|
68
63
|
remember(edges, {
|
|
69
64
|
from: server.id,
|
|
65
|
+
name: flow.name,
|
|
70
66
|
object: objectPath(server, flow.name),
|
|
71
67
|
side: 'out',
|
|
72
68
|
to: dest.id
|
|
@@ -80,6 +76,7 @@ export function edgesOf(document) {
|
|
|
80
76
|
remember(edges, {
|
|
81
77
|
api: flow.api,
|
|
82
78
|
from: source.id,
|
|
79
|
+
name: flow.name,
|
|
83
80
|
object: objectPath(source, flow.name),
|
|
84
81
|
side: 'in',
|
|
85
82
|
to: server.id
|
|
@@ -119,7 +116,7 @@ function covers(held, objectId) {
|
|
|
119
116
|
* direct({ fan_in: [], id: 'cell' });
|
|
120
117
|
*/
|
|
121
118
|
function direct(server) {
|
|
122
|
-
const names =
|
|
119
|
+
const names = publishedNames(server);
|
|
123
120
|
(server.fan_in || []).forEach((flow) => {
|
|
124
121
|
if (isAddress(flow.from)) {
|
|
125
122
|
names.push(flow.name);
|
|
@@ -116,3 +116,16 @@ export function profileFor(document, server) {
|
|
|
116
116
|
const values = { ...(profile.variables || {}), ...(server.profile.variables || {}) };
|
|
117
117
|
return { ...profile, variables: values };
|
|
118
118
|
}
|
|
119
|
+
|
|
120
|
+
/** Selects an explicitly named service or the sole service of the source. */
|
|
121
|
+
export function serviceOf(source, api) {
|
|
122
|
+
const listed = source.api || [];
|
|
123
|
+
if (!api && listed.length > 1) {
|
|
124
|
+
throw new Error(`system document server ${source.id} must name one api`);
|
|
125
|
+
}
|
|
126
|
+
const service = api ? listed.find((item) => { return item.id === api; }) : listed[0];
|
|
127
|
+
if (api && !service) {
|
|
128
|
+
throw new Error(`system document server ${source.id} provides no api ${api}`);
|
|
129
|
+
}
|
|
130
|
+
return service;
|
|
131
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isAddress } from './expandPattern.js';
|
|
2
|
-
import { clusterOf, objectPath, serverById } from './releaseNodes.js';
|
|
2
|
+
import { clusterOf, objectPath, serverById, serviceOf } from './releaseNodes.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Maps one module name onto a chart product.
|
|
@@ -16,7 +16,8 @@ function bindOne(name, spec, secret) {
|
|
|
16
16
|
return ['postgres', { enabled: spec.enabled !== false, secret: secret || 'scada-postgres' }];
|
|
17
17
|
}
|
|
18
18
|
if (name === 'message_bus') {
|
|
19
|
-
return ['rabbitmq', { enabled: spec.enabled !== false
|
|
19
|
+
return ['rabbitmq', { ...spec.chart, enabled: spec.enabled !== false,
|
|
20
|
+
...(spec.dead_letter ? { deadLetter: spec.dead_letter } : {}) }];
|
|
20
21
|
}
|
|
21
22
|
if (name === 'telemetry_store') {
|
|
22
23
|
return ['clickhouse', { enabled: spec.enabled !== false }];
|
|
@@ -102,19 +103,8 @@ export function machinesOf(server) {
|
|
|
102
103
|
* plantUrl({ api: [{ id: 'plant', url: 'http://192.0.2.10/api/v1' }], id: 'cell' });
|
|
103
104
|
*/
|
|
104
105
|
function plantUrl(source, api) {
|
|
105
|
-
const
|
|
106
|
-
|
|
107
|
-
return item.id === api;
|
|
108
|
-
})) {
|
|
109
|
-
throw new Error(`system document server ${source.id} provides no api ${api}`);
|
|
110
|
-
}
|
|
111
|
-
if (!api && listed.length > 1) {
|
|
112
|
-
throw new Error(`system document server ${source.id} must name one api`);
|
|
113
|
-
}
|
|
114
|
-
const plant = listed.find((item) => {
|
|
115
|
-
return item.id === 'plant';
|
|
116
|
-
});
|
|
117
|
-
return plant ? plant.url : '';
|
|
106
|
+
const service = serviceOf(source, api);
|
|
107
|
+
return service?.id === 'plant' ? service.url : '';
|
|
118
108
|
}
|
|
119
109
|
|
|
120
110
|
/**
|
|
@@ -18,7 +18,8 @@ function withImage(profile) {
|
|
|
18
18
|
if (!image) {
|
|
19
19
|
throw new Error('system document has no scada image');
|
|
20
20
|
}
|
|
21
|
-
return { image, ...
|
|
21
|
+
return { image, ...(app.site && { site: app.site }), ...(app.sink && { sink: app.sink }),
|
|
22
|
+
...bindModules(profile, app.secret) };
|
|
22
23
|
}
|
|
23
24
|
|
|
24
25
|
/**
|
|
@@ -109,6 +110,7 @@ export default function releases(document, cluster) {
|
|
|
109
110
|
const profile = profileFor(expanded, host);
|
|
110
111
|
assertHeld(expanded, links);
|
|
111
112
|
return frozen({
|
|
113
|
+
...(profile.target && { target: profile.target }),
|
|
112
114
|
apps: passedApps(profile),
|
|
113
115
|
scada: scadaRelease(expanded, host, profile, links)
|
|
114
116
|
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { assertHeld, edgesOf } from './releaseEdges.js';
|
|
2
|
+
import { serverById } from './releaseNodes.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Lists directed stream subscriptions for a source server in an expanded document.
|
|
6
|
+
* A receiver may explicitly select a service. A sender-only declaration has none.
|
|
7
|
+
*/
|
|
8
|
+
export default function subscriptions(document, sourceId) {
|
|
9
|
+
serverById(document, sourceId);
|
|
10
|
+
const edges = edgesOf(document);
|
|
11
|
+
assertHeld(document, edges);
|
|
12
|
+
const rows = [];
|
|
13
|
+
for (const { from, to, streams } of edges.filter((edge) => { return edge.from === sourceId; })) {
|
|
14
|
+
for (const [name, services] of streams) {
|
|
15
|
+
if (!services.size) { rows.push({ from, to, name }); }
|
|
16
|
+
services.forEach((api) => { rows.push({ from, to, name, api }); });
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return rows;
|
|
20
|
+
}
|
|
@@ -23,6 +23,10 @@ const appRoot = path.dirname(fileURLToPath(import.meta.url));
|
|
|
23
23
|
*/
|
|
24
24
|
export function readDeploymentConfig(env) {
|
|
25
25
|
const root = path.join(appRoot, '../..');
|
|
26
|
+
const prefetch = Number(env.STOMP_PREFETCH || '128');
|
|
27
|
+
if (!Number.isSafeInteger(prefetch) || prefetch < 1) {
|
|
28
|
+
throw new Error('STOMP_PREFETCH must be a positive safe integer');
|
|
29
|
+
}
|
|
26
30
|
return {
|
|
27
31
|
stomp: env.STOMP_URL,
|
|
28
32
|
postgres: env.POSTGRES_URL,
|
|
@@ -39,6 +43,8 @@ export function readDeploymentConfig(env) {
|
|
|
39
43
|
requestTimeoutMs: parseRequestTimeoutMs(env.REQUEST_TIMEOUT_MS),
|
|
40
44
|
pipelines: env.SINK_PIPELINES !== 'false',
|
|
41
45
|
pipeline: {
|
|
46
|
+
prefetch,
|
|
47
|
+
deadLetter: env.SEGMENTS_DEAD_LETTER_ENABLED === 'true',
|
|
42
48
|
size: parseInt(env.BATCH_SIZE || '50', 10),
|
|
43
49
|
interval: parseInt(env.FLUSH_INTERVAL || '2', 10),
|
|
44
50
|
threshold: parseInt(env.CIRCUIT_THRESHOLD || '5', 10),
|
|
@@ -40,7 +40,7 @@ function isHeartbeat(parsed) {
|
|
|
40
40
|
* @param {object} codec - downstream codec
|
|
41
41
|
* @returns {Promise<void>}
|
|
42
42
|
*/
|
|
43
|
-
async function flushLane(lane, codec) {
|
|
43
|
+
async function flushLane(lane, codec, deadLetter) {
|
|
44
44
|
const batch = lane.pending;
|
|
45
45
|
if (!batch) {
|
|
46
46
|
return;
|
|
@@ -57,9 +57,11 @@ async function flushLane(lane, codec) {
|
|
|
57
57
|
}
|
|
58
58
|
} catch (error) {
|
|
59
59
|
for (const settle of batch.settles) {
|
|
60
|
-
settle.nack();
|
|
60
|
+
settle.nack(!(deadLetter && error.permanent));
|
|
61
|
+
}
|
|
62
|
+
if (!error.permanent) {
|
|
63
|
+
throw error;
|
|
61
64
|
}
|
|
62
|
-
throw error;
|
|
63
65
|
}
|
|
64
66
|
}
|
|
65
67
|
|
|
@@ -78,7 +80,12 @@ async function bufferHeartbeat(ctx, envelope, parsed) {
|
|
|
78
80
|
lane.pending.payload = envelope.payload;
|
|
79
81
|
lane.pending.settles.push(envelope.settle);
|
|
80
82
|
} else {
|
|
81
|
-
|
|
83
|
+
try {
|
|
84
|
+
await api.flush(lane);
|
|
85
|
+
} catch (error) {
|
|
86
|
+
envelope.settle.nack(true);
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
82
89
|
const pending = {
|
|
83
90
|
key,
|
|
84
91
|
destination: envelope.destination,
|
|
@@ -88,7 +95,7 @@ async function bufferHeartbeat(ctx, envelope, parsed) {
|
|
|
88
95
|
const timer = setTimeout(() => {
|
|
89
96
|
lane.queue = lane.queue.then(() => {
|
|
90
97
|
return api.flush(lane);
|
|
91
|
-
});
|
|
98
|
+
}).catch(() => { return undefined; });
|
|
92
99
|
}, intervalMs);
|
|
93
100
|
Object.assign(lane, { pending, timer });
|
|
94
101
|
}
|
|
@@ -110,12 +117,12 @@ async function step(ctx, envelope, parsed) {
|
|
|
110
117
|
await bufferHeartbeat(ctx, envelope, parsed);
|
|
111
118
|
return;
|
|
112
119
|
}
|
|
113
|
-
await ctx.api.flush(ctx.lane);
|
|
114
120
|
try {
|
|
121
|
+
await ctx.api.flush(ctx.lane);
|
|
115
122
|
await ctx.codec.accept({ destination: envelope.destination, payload: envelope.payload });
|
|
116
123
|
envelope.settle.ack();
|
|
117
124
|
} catch (error) {
|
|
118
|
-
envelope.settle.nack();
|
|
125
|
+
envelope.settle.nack(!(ctx.deadLetter && error.permanent));
|
|
119
126
|
throw error;
|
|
120
127
|
}
|
|
121
128
|
}
|
|
@@ -148,9 +155,18 @@ export default function segmentCoalesce(codec, options = {}) {
|
|
|
148
155
|
const lanes = new Map();
|
|
149
156
|
const api = {
|
|
150
157
|
accept(envelope) {
|
|
151
|
-
|
|
158
|
+
let parsed;
|
|
159
|
+
try {
|
|
160
|
+
parsed = JSON.parse(envelope.payload);
|
|
161
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
162
|
+
throw new Error('Segment must be an object');
|
|
163
|
+
}
|
|
164
|
+
} catch {
|
|
165
|
+
envelope.settle.nack(!options.deadLetter);
|
|
166
|
+
return Promise.resolve();
|
|
167
|
+
}
|
|
152
168
|
const lane = laneState(lanes, parsed.machine);
|
|
153
|
-
const ctx = { api, lane, codec, size, intervalMs };
|
|
169
|
+
const ctx = { api, lane, codec, size, intervalMs, deadLetter: options.deadLetter };
|
|
154
170
|
lane.queue = lane.queue.then(() => {
|
|
155
171
|
return step(ctx, envelope, parsed);
|
|
156
172
|
}).catch(() => {
|
|
@@ -159,7 +175,7 @@ export default function segmentCoalesce(codec, options = {}) {
|
|
|
159
175
|
return lane.queue;
|
|
160
176
|
},
|
|
161
177
|
async flush(lane) {
|
|
162
|
-
await flushLane(lane, codec);
|
|
178
|
+
await flushLane(lane, codec, options.deadLetter);
|
|
163
179
|
}
|
|
164
180
|
};
|
|
165
181
|
return api;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { execFileSync } from 'node:child_process';
|
|
2
|
+
|
|
3
|
+
/** Validates the target before helm or kubectl can fall back to a default. */
|
|
4
|
+
function validate(plan, action) {
|
|
5
|
+
if (!['render', 'check', 'diff', 'apply', 'adopt', 'check-adoption'].includes(action)) {
|
|
6
|
+
throw new Error('action must be render, check, diff or apply (adopt/check-adoption for migration)');
|
|
7
|
+
}
|
|
8
|
+
const fields = action === 'render' ? ['chart', 'release', 'namespace'] : ['chart', 'release', 'namespace', 'context', 'fieldManager'];
|
|
9
|
+
fields.forEach((key) => {
|
|
10
|
+
if (typeof plan[key] !== 'string' || !plan[key].trim()) { throw new Error(`release requires ${key}`); }
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Every Kubernetes command, including the post-rollout hook, shares this target. */
|
|
15
|
+
function kubeAt(plan, execute) {
|
|
16
|
+
return (args, input) => {
|
|
17
|
+
return execute('kubectl', ['--context', plan.context, '--namespace', plan.namespace, ...args],
|
|
18
|
+
{ input, encoding: 'utf8', stdio: ['pipe', 'inherit', 'inherit'] });
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function applyWith(kube, manifests, flags, checkOnly) {
|
|
23
|
+
const args = ['apply', ...flags, '-f', '-'];
|
|
24
|
+
kube([...args, '--dry-run=server'], manifests);
|
|
25
|
+
if (!checkOnly) { kube(args, manifests); }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Adopt legacy fields, then enforce the complete manifest before waiting for rollouts. */
|
|
29
|
+
function applyRelease(plan, action, kube, manifests) {
|
|
30
|
+
if (['adopt', 'check-adoption'].includes(action)) {
|
|
31
|
+
applyWith(kube, manifests, ['--field-manager=kubectl-client-side-apply'], action === 'check-adoption');
|
|
32
|
+
if (action === 'check-adoption') { return ''; }
|
|
33
|
+
}
|
|
34
|
+
applyWith(kube, manifests, ['--server-side', `--field-manager=${plan.fieldManager}`], action === 'check');
|
|
35
|
+
if (action === 'check') { return ''; }
|
|
36
|
+
for (const workload of plan.workloads || []) {
|
|
37
|
+
kube(['rollout', 'status', workload, '--timeout=300s']);
|
|
38
|
+
}
|
|
39
|
+
plan.afterReady?.(plan, kube);
|
|
40
|
+
return '';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Renders a Helm release and executes the selected kubectl action synchronously.
|
|
45
|
+
* Supply chart, release, namespace and values. Cluster actions additionally require
|
|
46
|
+
* an explicit context and fieldManager. prerequisites and workloads are kubectl
|
|
47
|
+
* resource/name references. afterReady(plan, kube) runs after a successful apply.
|
|
48
|
+
*/
|
|
49
|
+
export default function executeRelease(plan, action = 'render', execute = execFileSync) {
|
|
50
|
+
validate(plan, action);
|
|
51
|
+
const manifests = execute('helm', ['template', plan.release, plan.chart, '--namespace', plan.namespace, '-f', '-'],
|
|
52
|
+
{ input: JSON.stringify(plan.values), encoding: 'utf8' });
|
|
53
|
+
if (action === 'render') { return manifests; }
|
|
54
|
+
const kube = kubeAt(plan, execute);
|
|
55
|
+
for (const resource of plan.prerequisites || []) { kube(['get', resource, '-o', 'name']); }
|
|
56
|
+
if (action === 'diff') {
|
|
57
|
+
return kube(['diff', '--server-side', `--field-manager=${plan.fieldManager}`, '-f', '-'], manifests);
|
|
58
|
+
}
|
|
59
|
+
return applyRelease(plan, action, kube, manifests);
|
|
60
|
+
}
|
|
@@ -32,12 +32,19 @@ export default function segmentCodec(collector) {
|
|
|
32
32
|
return {
|
|
33
33
|
async accept(raw) {
|
|
34
34
|
try {
|
|
35
|
-
|
|
35
|
+
let parsed;
|
|
36
|
+
let record;
|
|
37
|
+
try {
|
|
38
|
+
parsed = JSON.parse(raw.payload);
|
|
39
|
+
record = parsed.type === 'split-family' ? familyRecord(parsed) : segmentNormalize(parsed);
|
|
40
|
+
} catch (error) {
|
|
41
|
+
error.permanent = true;
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
36
44
|
if (parsed.type === 'split-family') {
|
|
37
|
-
await collector.accept(
|
|
45
|
+
await collector.accept(record);
|
|
38
46
|
return;
|
|
39
47
|
}
|
|
40
|
-
const record = segmentNormalize(parsed);
|
|
41
48
|
await collector.accept(record);
|
|
42
49
|
} catch (error) {
|
|
43
50
|
processingErrorLog('segment_codec', error, { destination: raw.destination, payload: raw.payload });
|
|
@@ -60,6 +60,7 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
|
|
|
60
60
|
const dispatch = segmentDispatch(continued, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
|
|
61
61
|
const coalesce = segmentCoalesce(segmentCodec(dispatch), {
|
|
62
62
|
size: config.size,
|
|
63
|
+
deadLetter: config.deadLetter,
|
|
63
64
|
interval: config.interval
|
|
64
65
|
});
|
|
65
66
|
const destination = config.segmentsDestination || segmentsIngestDestination;
|
|
@@ -68,7 +69,8 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
|
|
|
68
69
|
passcode: config.passcode,
|
|
69
70
|
host: config.host,
|
|
70
71
|
serial: false,
|
|
71
|
-
manualAck: true
|
|
72
|
+
manualAck: true,
|
|
73
|
+
prefetch: config.prefetch
|
|
72
74
|
});
|
|
73
75
|
const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
|
|
74
76
|
intervalMs: (config.poll || 5) * 1000
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Builds durable topic exchanges, queues and AMQP 0-9-1 outboxes for one virtual host. */
|
|
2
|
+
export default function rabbitmqDefinitions({ vhost, users = [], permissions = [], exchanges = [],
|
|
3
|
+
queues = [], bindings = [], outboxes = [] }) {
|
|
4
|
+
if (!vhost) { throw new Error('a virtual host is required'); }
|
|
5
|
+
const routes = outboxes.flatMap(({ exchange, queue, keys }) => {
|
|
6
|
+
return keys.map((key) => { return { source: exchange, destination: queue, key }; });
|
|
7
|
+
});
|
|
8
|
+
const names = [...queues, ...outboxes.map(({ queue }) => { return queue; })];
|
|
9
|
+
return { vhosts: [{ name: vhost }], users, permissions,
|
|
10
|
+
exchanges: exchanges.map((name) => { return { name, vhost, type: 'topic', durable: true, auto_delete: false }; }),
|
|
11
|
+
queues: [...new Set(names)].map((name) => { return { name, vhost, durable: true, auto_delete: false, arguments: {} }; }),
|
|
12
|
+
bindings: [...bindings, ...routes].map(({ source, destination, key }) => {
|
|
13
|
+
return { source, vhost, destination, destination_type: 'queue', routing_key: key, arguments: {} };
|
|
14
|
+
}),
|
|
15
|
+
parameters: outboxes.filter(({ shovel }) => { return shovel; }).map(({ queue, shovel }) => {
|
|
16
|
+
return { vhost, component: 'shovel', name: shovel.name, value: {
|
|
17
|
+
'src-protocol': 'amqp091', 'src-uri': shovel.sourceUri, 'src-queue': queue,
|
|
18
|
+
'dest-protocol': 'amqp091', 'dest-uri': shovel.destinationUri,
|
|
19
|
+
'dest-exchange': shovel.destinationExchange, 'reconnect-delay': shovel.reconnectDelay,
|
|
20
|
+
'dest-publish-properties': { delivery_mode: 2 }
|
|
21
|
+
} };
|
|
22
|
+
})
|
|
23
|
+
};
|
|
24
|
+
}
|