@yarkivaev/scada 2.3.113 → 2.3.115

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/README.md +53 -0
  2. package/charts/scada/Chart.yaml +2 -2
  3. package/charts/scada/templates/_helpers.tpl +12 -4
  4. package/charts/scada/templates/clickhouse.yaml +3 -3
  5. package/charts/scada/templates/collector.yaml +2 -2
  6. package/charts/scada/templates/namespace.yaml +6 -0
  7. package/charts/scada/templates/postgres.yaml +3 -3
  8. package/charts/scada/templates/processes.yaml +88 -0
  9. package/charts/scada/templates/rabbitmq-config.yaml +91 -0
  10. package/charts/scada/templates/rabbitmq.yaml +54 -107
  11. package/charts/scada/templates/site-server.yaml +5 -3
  12. package/charts/scada/templates/supervisor-sink.yaml +8 -2
  13. package/charts/scada/values.schema.json +22 -0
  14. package/charts/scada/values.yaml +22 -0
  15. package/index.js +3 -0
  16. package/package.json +5 -3
  17. package/src/application/deploy/collectorProcesses.js +42 -0
  18. package/src/application/deploy/expandFlows.js +21 -10
  19. package/src/application/deploy/publishedStreams.js +30 -0
  20. package/src/application/deploy/releaseEdges.js +13 -16
  21. package/src/application/deploy/releaseNodes.js +13 -0
  22. package/src/application/deploy/releaseValues.js +5 -15
  23. package/src/application/deploy/releases.js +3 -1
  24. package/src/application/deploy/subscriptions.js +20 -0
  25. package/src/application/supervisorSink.js +6 -0
  26. package/src/domain/segment/segmentCoalesce.js +26 -10
  27. package/src/infrastructure/deploy/executeRelease.js +60 -0
  28. package/src/infrastructure/ingest/codecs/segmentCodec.js +10 -3
  29. package/src/infrastructure/ingest/pipelines/segmentPipeline.js +3 -1
  30. package/src/infrastructure/messaging/rabbitmq/definitions.js +24 -0
  31. package/src/infrastructure/messaging/rabbitmq/reconcile.js +104 -0
@@ -1,4 +1,15 @@
1
1
  role: edge
2
+ # Override the resource name prefix when adopting an existing installation.
3
+ prefix: ""
4
+ createNamespace: false
5
+ site:
6
+ enabled: true
7
+ sink:
8
+ enabled: true
9
+ # Labels for workloads that must stay outside immutable selectors.
10
+ workloadLabels: {}
11
+ # Named deployments with command, env, optional resources and hostDevices.
12
+ processes: []
2
13
  image: ""
3
14
  command: []
4
15
  pullSecret: ""
@@ -21,6 +32,17 @@ postgres:
21
32
  data: /var/lib/postgresql/data/pgdata
22
33
  rabbitmq:
23
34
  enabled: true
35
+ deadLetter: null
36
+ # A named standalone PVC replaces the default volumeClaimTemplates when set.
37
+ claimName: ""
38
+ configName: ""
39
+ rolloutOnConfigChange: false
40
+ definitions: null
41
+ plugins: [rabbitmq_stomp, rabbitmq_mqtt, rabbitmq_management]
42
+ extraConfig: {}
43
+ resources: {}
44
+ readinessProbe: {}
45
+ livenessProbe: {}
24
46
  image: rabbitmq:3.13-management
25
47
  storage: 5Gi
26
48
  storageClass: ""
package/index.js CHANGED
@@ -96,9 +96,12 @@ export { default as analyticsMcp } from './src/application/mcp/analyticsMcp.js';
96
96
  export { default as expand } from './src/application/deploy/expand.js';
97
97
  export { default as readDocument } from './src/application/deploy/readDocument.js';
98
98
  export { default as mergeDocuments } from './src/application/deploy/mergeDocuments.js';
99
+ export { default as subscriptions } from './src/application/deploy/subscriptions.js';
99
100
  export { default as releases } from './src/application/deploy/releases.js';
100
101
  export { default as paths } from './src/application/deploy/paths.js';
101
102
  export { default as appReleases } from './src/application/deploy/appReleases.js';
102
103
  export { default as withCollectors } from './src/application/deploy/withCollectors.js';
103
104
  export { default as pointPaths } from './src/application/deploy/pointPaths.js';
104
105
  export { clusterOf, serverFor } from './src/application/deploy/releaseNodes.js';
106
+ export { default as executeRelease } from './src/infrastructure/deploy/executeRelease.js';
107
+ export { default as collectorProcesses } from './src/application/deploy/collectorProcesses.js';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yarkivaev/scada",
3
- "version": "2.3.113",
3
+ "version": "2.3.115",
4
4
  "description": "SCADA domain objects, state persistence, and plant monitoring",
5
5
  "repository": {
6
6
  "type": "git",
@@ -13,7 +13,9 @@
13
13
  "./client": "./src/infrastructure/client/index.js",
14
14
  "./stateDataFake": "./src/infrastructure/persistence/stateDataFromMemory.js",
15
15
  "./startTestEdgeApi": "./src/infrastructure/http/edge/startTestEdgeApi.js",
16
- "./package.json": "./package.json"
16
+ "./package.json": "./package.json",
17
+ "./rabbitmq/definitions": "./src/infrastructure/messaging/rabbitmq/definitions.js",
18
+ "./rabbitmq/reconcile": "./src/infrastructure/messaging/rabbitmq/reconcile.js"
17
19
  },
18
20
  "bin": {
19
21
  "modbus-to-mqtt": "./src/bin/modbus-to-mqtt.js",
@@ -31,7 +33,7 @@
31
33
  "@clickhouse/client": "^1.0.0",
32
34
  "@modelcontextprotocol/sdk": "^1.24.3",
33
35
  "@yarkivaev/simple-server": "1.0.2",
34
- "@yarkivaev/source-to-sink": "1.3.8",
36
+ "@yarkivaev/source-to-sink": "1.3.10",
35
37
  "amqplib": "^0.10.9",
36
38
  "pg": "^8.0.0",
37
39
  "stompit": "^1.0.0",
@@ -0,0 +1,42 @@
1
+ import { isDeepStrictEqual } from 'node:util';
2
+
3
+ /** Group rows in first-seen order, keeping source order within each process. */
4
+ function grouped(machines, describe) {
5
+ const groups = new Map();
6
+ const claimed = new Set();
7
+ for (const row of machines) {
8
+ if (claimed.has(row.name)) { throw new Error(`duplicate source ${row.name}`); }
9
+ claimed.add(row.name);
10
+ const descriptor = describe(row);
11
+ if (typeof descriptor?.name !== 'string' || !descriptor.name.trim()) {
12
+ throw new Error(`unassigned source ${row.name}`);
13
+ }
14
+ const group = groups.get(descriptor.name) || { descriptor, rows: [] };
15
+ if (!isDeepStrictEqual(group.descriptor, descriptor)) {
16
+ throw new Error(`conflicting descriptions for process ${descriptor.name}`);
17
+ }
18
+ group.rows.push(row);
19
+ groups.set(descriptor.name, group);
20
+ }
21
+ return [...groups.values()];
22
+ }
23
+
24
+ /**
25
+ * describe(row) returns a process descriptor with a name, command and optional
26
+ * env and resources. All sources describing the same process must agree.
27
+ * project(descriptor, rows) supplies adapter-specific env and optional hostDevices.
28
+ * A physical device may appear in many source rows but belongs to only one process.
29
+ */
30
+ export default function collectorProcesses(machines, describe, project) {
31
+ const devices = new Set();
32
+ return grouped(machines, describe).map(({ descriptor, rows }) => {
33
+ const settings = project(descriptor, rows);
34
+ const hostDevices = [...new Set(settings.hostDevices || [])].sort();
35
+ hostDevices.forEach((device) => {
36
+ if (devices.has(device)) { throw new Error(`serial port or device ${device} is shared between processes`); }
37
+ devices.add(device);
38
+ });
39
+ return { name: descriptor.name, command: descriptor.command, env: { ...descriptor.env, ...settings.env },
40
+ hostDevices, resources: descriptor.resources };
41
+ });
42
+ }
@@ -1,4 +1,6 @@
1
1
  import { assertLeaf, deviceFlows, expandObjects } from './expandDevices.js';
2
+ import { availableNames, expandApis, publishedNames } from './publishedStreams.js';
3
+ import { serviceOf } from './releaseNodes.js';
2
4
  import { isAddress, picked } from './expandPattern.js';
3
5
 
4
6
  /**
@@ -32,18 +34,17 @@ function chosen(pattern, pool, strict) {
32
34
  * @example
33
35
  * poolOf({ fanIn: [], fanOut: [] }, 'archive');
34
36
  */
35
- function poolOf(source, receiverId) {
37
+ function poolOf(source, receiverId, api) {
38
+ const available = availableNames(source, api);
36
39
  const aimed = source.fanOut.filter((flow) => {
37
40
  return flow.to === receiverId;
38
41
  }).map((flow) => {
39
42
  return flow.name;
40
43
  });
41
44
  if (aimed.length > 0) {
42
- return aimed;
45
+ return aimed.filter((name) => { return available.includes(name); });
43
46
  }
44
- return source.fanIn.map((flow) => {
45
- return flow.name;
46
- });
47
+ return available;
47
48
  }
48
49
 
49
50
  /**
@@ -60,9 +61,7 @@ function poolOf(source, receiverId) {
60
61
  */
61
62
  function expandOut(server, ids, strict) {
62
63
  const rows = [];
63
- const streams = server.fanIn.map((flow) => {
64
- return flow.name;
65
- });
64
+ const streams = availableNames(server);
66
65
  server.patternsOut.forEach((flow) => {
67
66
  const destinations = picked(flow.to, ids, 'server', true);
68
67
  const names = picked(flow.name, streams, 'path', strict);
@@ -76,6 +75,18 @@ function expandOut(server, ids, strict) {
76
75
  return rows;
77
76
  }
78
77
 
78
+ /* A fixed object path can be held implicitly, but an API-only stream requires its service. */
79
+ function incomingNames(source, receiver, flow, strict) {
80
+ const api = serviceOf(source, flow.api)?.id;
81
+ const pool = poolOf(source, receiver, api);
82
+ const names = chosen(flow.name, pool, strict);
83
+ if (strict && api && publishedNames(source).includes(flow.name)
84
+ && !availableNames(source, api).includes(flow.name)) {
85
+ throw new Error(`system document stream ${flow.name} is unavailable on api ${flow.api}`);
86
+ }
87
+ return names;
88
+ }
89
+
79
90
  /**
80
91
  * Expands fan-in whose source is a server.
81
92
  *
@@ -101,7 +112,7 @@ function expandIn(server, drafts, strict) {
101
112
  const source = drafts.find((item) => {
102
113
  return item.id === sourceId;
103
114
  });
104
- chosen(flow.name, poolOf(source, server.id), strict).forEach((path) => {
115
+ incomingNames(source, server.id, flow, strict).forEach((path) => {
105
116
  assertLeaf(server.objects, path);
106
117
  const copy = { from: sourceId, name: path };
107
118
  if (flow.api) {
@@ -144,7 +155,7 @@ function draftOne(server, paths, mappings) {
144
155
  remote.push(flow);
145
156
  });
146
157
  return {
147
- api: server.api,
158
+ api: expandApis(server.api, paths, objects),
148
159
  fanIn: local,
149
160
  id: server.id,
150
161
  objects,
@@ -0,0 +1,30 @@
1
+ import { assertLeaf } from './expandDevices.js';
2
+ import { picked } from './expandPattern.js';
3
+
4
+ /** Resolve each API's published stream names, preserving its opaque contract metadata. */
5
+ export function expandApis(apis, paths, objects) {
6
+ return apis?.map((api) => {
7
+ if (!api.publishes) { return api; }
8
+ const publishes = api.publishes.flatMap((entry) => {
9
+ return picked(entry.name, paths, 'path', true).map((name) => {
10
+ assertLeaf(objects, name);
11
+ return { ...entry, name };
12
+ });
13
+ });
14
+ return { ...api, publishes };
15
+ });
16
+ }
17
+
18
+ /** Streams made available by publishers on the selected service. */
19
+ export function publishedNames(server, api) {
20
+ return (server.api || []).filter((item) => {
21
+ return !api || item.id === api;
22
+ }).flatMap((item) => {
23
+ return (item.publishes || []).map(({ name }) => { return name; });
24
+ });
25
+ }
26
+
27
+ /** Locally collected and forwarded streams stay available through the server's services. */
28
+ export function availableNames(source, api) {
29
+ return [...new Set([...source.fanIn.map(({ name }) => { return name; }), ...publishedNames(source, api)])];
30
+ }
@@ -1,3 +1,4 @@
1
+ import { publishedNames } from './publishedStreams.js';
1
2
  import { isAddress } from './expandPattern.js';
2
3
  import { asPath, objectPath, serverById } from './releaseNodes.js';
3
4
 
@@ -28,28 +29,22 @@ function sameObjects(left, right) {
28
29
  * remember([], { from: 'cell', object: 'unit', side: 'out', to: 'archive' });
29
30
  */
30
31
  function remember(edges, edge) {
31
- const found = edges.find((item) => {
32
+ let found = edges.find((item) => {
32
33
  return item.from === edge.from && item.to === edge.to;
33
34
  });
34
35
  if (!found) {
35
- edges.push({
36
- api: edge.api,
37
- from: edge.from,
38
- objects: [edge.object],
39
- sides: { [edge.side]: [edge.object] },
40
- to: edge.to
41
- });
42
- return;
43
- }
44
- const prior = found.sides[edge.side] ? found.sides[edge.side].slice() : [];
45
- if (!prior.includes(edge.object)) {
46
- prior.push(edge.object);
36
+ found = { from: edge.from, to: edge.to, sides: {}, streams: new Map() };
37
+ edges.push(found);
47
38
  }
48
- found.sides[edge.side] = prior;
49
- found.objects = (found.sides.out || found.sides.in).slice();
39
+ found.sides[edge.side] ||= [];
40
+ const objects = found.sides[edge.side];
41
+ if (!objects.includes(edge.object)) { objects.push(edge.object); }
42
+ const services = found.streams.get(edge.name) || new Set();
50
43
  if (edge.api) {
44
+ services.add(edge.api);
51
45
  found.api = edge.api;
52
46
  }
47
+ found.streams.set(edge.name, services);
53
48
  }
54
49
 
55
50
  /**
@@ -67,6 +62,7 @@ export function edgesOf(document) {
67
62
  const dest = serverById(document, flow.to);
68
63
  remember(edges, {
69
64
  from: server.id,
65
+ name: flow.name,
70
66
  object: objectPath(server, flow.name),
71
67
  side: 'out',
72
68
  to: dest.id
@@ -80,6 +76,7 @@ export function edgesOf(document) {
80
76
  remember(edges, {
81
77
  api: flow.api,
82
78
  from: source.id,
79
+ name: flow.name,
83
80
  object: objectPath(source, flow.name),
84
81
  side: 'in',
85
82
  to: server.id
@@ -119,7 +116,7 @@ function covers(held, objectId) {
119
116
  * direct({ fan_in: [], id: 'cell' });
120
117
  */
121
118
  function direct(server) {
122
- const names = [];
119
+ const names = publishedNames(server);
123
120
  (server.fan_in || []).forEach((flow) => {
124
121
  if (isAddress(flow.from)) {
125
122
  names.push(flow.name);
@@ -116,3 +116,16 @@ export function profileFor(document, server) {
116
116
  const values = { ...(profile.variables || {}), ...(server.profile.variables || {}) };
117
117
  return { ...profile, variables: values };
118
118
  }
119
+
120
+ /** Selects an explicitly named service or the sole service of the source. */
121
+ export function serviceOf(source, api) {
122
+ const listed = source.api || [];
123
+ if (!api && listed.length > 1) {
124
+ throw new Error(`system document server ${source.id} must name one api`);
125
+ }
126
+ const service = api ? listed.find((item) => { return item.id === api; }) : listed[0];
127
+ if (api && !service) {
128
+ throw new Error(`system document server ${source.id} provides no api ${api}`);
129
+ }
130
+ return service;
131
+ }
@@ -1,5 +1,5 @@
1
1
  import { isAddress } from './expandPattern.js';
2
- import { clusterOf, objectPath, serverById } from './releaseNodes.js';
2
+ import { clusterOf, objectPath, serverById, serviceOf } from './releaseNodes.js';
3
3
 
4
4
  /**
5
5
  * Maps one module name onto a chart product.
@@ -16,7 +16,8 @@ function bindOne(name, spec, secret) {
16
16
  return ['postgres', { enabled: spec.enabled !== false, secret: secret || 'scada-postgres' }];
17
17
  }
18
18
  if (name === 'message_bus') {
19
- return ['rabbitmq', { enabled: spec.enabled !== false }];
19
+ return ['rabbitmq', { ...spec.chart, enabled: spec.enabled !== false,
20
+ ...(spec.dead_letter ? { deadLetter: spec.dead_letter } : {}) }];
20
21
  }
21
22
  if (name === 'telemetry_store') {
22
23
  return ['clickhouse', { enabled: spec.enabled !== false }];
@@ -102,19 +103,8 @@ export function machinesOf(server) {
102
103
  * plantUrl({ api: [{ id: 'plant', url: 'http://192.0.2.10/api/v1' }], id: 'cell' });
103
104
  */
104
105
  function plantUrl(source, api) {
105
- const listed = source.api || [];
106
- if (api && !listed.some((item) => {
107
- return item.id === api;
108
- })) {
109
- throw new Error(`system document server ${source.id} provides no api ${api}`);
110
- }
111
- if (!api && listed.length > 1) {
112
- throw new Error(`system document server ${source.id} must name one api`);
113
- }
114
- const plant = listed.find((item) => {
115
- return item.id === 'plant';
116
- });
117
- return plant ? plant.url : '';
106
+ const service = serviceOf(source, api);
107
+ return service?.id === 'plant' ? service.url : '';
118
108
  }
119
109
 
120
110
  /**
@@ -18,7 +18,8 @@ function withImage(profile) {
18
18
  if (!image) {
19
19
  throw new Error('system document has no scada image');
20
20
  }
21
- return { image, ...bindModules(profile, app.secret) };
21
+ return { image, ...(app.site && { site: app.site }), ...(app.sink && { sink: app.sink }),
22
+ ...bindModules(profile, app.secret) };
22
23
  }
23
24
 
24
25
  /**
@@ -109,6 +110,7 @@ export default function releases(document, cluster) {
109
110
  const profile = profileFor(expanded, host);
110
111
  assertHeld(expanded, links);
111
112
  return frozen({
113
+ ...(profile.target && { target: profile.target }),
112
114
  apps: passedApps(profile),
113
115
  scada: scadaRelease(expanded, host, profile, links)
114
116
  });
@@ -0,0 +1,20 @@
1
+ import { assertHeld, edgesOf } from './releaseEdges.js';
2
+ import { serverById } from './releaseNodes.js';
3
+
4
+ /**
5
+ * Lists directed stream subscriptions for a source server in an expanded document.
6
+ * A receiver may explicitly select a service. A sender-only declaration has none.
7
+ */
8
+ export default function subscriptions(document, sourceId) {
9
+ serverById(document, sourceId);
10
+ const edges = edgesOf(document);
11
+ assertHeld(document, edges);
12
+ const rows = [];
13
+ for (const { from, to, streams } of edges.filter((edge) => { return edge.from === sourceId; })) {
14
+ for (const [name, services] of streams) {
15
+ if (!services.size) { rows.push({ from, to, name }); }
16
+ services.forEach((api) => { rows.push({ from, to, name, api }); });
17
+ }
18
+ }
19
+ return rows;
20
+ }
@@ -23,6 +23,10 @@ const appRoot = path.dirname(fileURLToPath(import.meta.url));
23
23
  */
24
24
  export function readDeploymentConfig(env) {
25
25
  const root = path.join(appRoot, '../..');
26
+ const prefetch = Number(env.STOMP_PREFETCH || '128');
27
+ if (!Number.isSafeInteger(prefetch) || prefetch < 1) {
28
+ throw new Error('STOMP_PREFETCH must be a positive safe integer');
29
+ }
26
30
  return {
27
31
  stomp: env.STOMP_URL,
28
32
  postgres: env.POSTGRES_URL,
@@ -39,6 +43,8 @@ export function readDeploymentConfig(env) {
39
43
  requestTimeoutMs: parseRequestTimeoutMs(env.REQUEST_TIMEOUT_MS),
40
44
  pipelines: env.SINK_PIPELINES !== 'false',
41
45
  pipeline: {
46
+ prefetch,
47
+ deadLetter: env.SEGMENTS_DEAD_LETTER_ENABLED === 'true',
42
48
  size: parseInt(env.BATCH_SIZE || '50', 10),
43
49
  interval: parseInt(env.FLUSH_INTERVAL || '2', 10),
44
50
  threshold: parseInt(env.CIRCUIT_THRESHOLD || '5', 10),
@@ -40,7 +40,7 @@ function isHeartbeat(parsed) {
40
40
  * @param {object} codec - downstream codec
41
41
  * @returns {Promise<void>}
42
42
  */
43
- async function flushLane(lane, codec) {
43
+ async function flushLane(lane, codec, deadLetter) {
44
44
  const batch = lane.pending;
45
45
  if (!batch) {
46
46
  return;
@@ -57,9 +57,11 @@ async function flushLane(lane, codec) {
57
57
  }
58
58
  } catch (error) {
59
59
  for (const settle of batch.settles) {
60
- settle.nack();
60
+ settle.nack(!(deadLetter && error.permanent));
61
+ }
62
+ if (!error.permanent) {
63
+ throw error;
61
64
  }
62
- throw error;
63
65
  }
64
66
  }
65
67
 
@@ -78,7 +80,12 @@ async function bufferHeartbeat(ctx, envelope, parsed) {
78
80
  lane.pending.payload = envelope.payload;
79
81
  lane.pending.settles.push(envelope.settle);
80
82
  } else {
81
- await api.flush(lane);
83
+ try {
84
+ await api.flush(lane);
85
+ } catch (error) {
86
+ envelope.settle.nack(true);
87
+ throw error;
88
+ }
82
89
  const pending = {
83
90
  key,
84
91
  destination: envelope.destination,
@@ -88,7 +95,7 @@ async function bufferHeartbeat(ctx, envelope, parsed) {
88
95
  const timer = setTimeout(() => {
89
96
  lane.queue = lane.queue.then(() => {
90
97
  return api.flush(lane);
91
- });
98
+ }).catch(() => { return undefined; });
92
99
  }, intervalMs);
93
100
  Object.assign(lane, { pending, timer });
94
101
  }
@@ -110,12 +117,12 @@ async function step(ctx, envelope, parsed) {
110
117
  await bufferHeartbeat(ctx, envelope, parsed);
111
118
  return;
112
119
  }
113
- await ctx.api.flush(ctx.lane);
114
120
  try {
121
+ await ctx.api.flush(ctx.lane);
115
122
  await ctx.codec.accept({ destination: envelope.destination, payload: envelope.payload });
116
123
  envelope.settle.ack();
117
124
  } catch (error) {
118
- envelope.settle.nack();
125
+ envelope.settle.nack(!(ctx.deadLetter && error.permanent));
119
126
  throw error;
120
127
  }
121
128
  }
@@ -148,9 +155,18 @@ export default function segmentCoalesce(codec, options = {}) {
148
155
  const lanes = new Map();
149
156
  const api = {
150
157
  accept(envelope) {
151
- const parsed = JSON.parse(envelope.payload);
158
+ let parsed;
159
+ try {
160
+ parsed = JSON.parse(envelope.payload);
161
+ if (!parsed || typeof parsed !== 'object') {
162
+ throw new Error('Segment must be an object');
163
+ }
164
+ } catch {
165
+ envelope.settle.nack(!options.deadLetter);
166
+ return Promise.resolve();
167
+ }
152
168
  const lane = laneState(lanes, parsed.machine);
153
- const ctx = { api, lane, codec, size, intervalMs };
169
+ const ctx = { api, lane, codec, size, intervalMs, deadLetter: options.deadLetter };
154
170
  lane.queue = lane.queue.then(() => {
155
171
  return step(ctx, envelope, parsed);
156
172
  }).catch(() => {
@@ -159,7 +175,7 @@ export default function segmentCoalesce(codec, options = {}) {
159
175
  return lane.queue;
160
176
  },
161
177
  async flush(lane) {
162
- await flushLane(lane, codec);
178
+ await flushLane(lane, codec, options.deadLetter);
163
179
  }
164
180
  };
165
181
  return api;
@@ -0,0 +1,60 @@
1
+ import { execFileSync } from 'node:child_process';
2
+
3
+ /** Validates the target before helm or kubectl can fall back to a default. */
4
+ function validate(plan, action) {
5
+ if (!['render', 'check', 'diff', 'apply', 'adopt', 'check-adoption'].includes(action)) {
6
+ throw new Error('action must be render, check, diff or apply (adopt/check-adoption for migration)');
7
+ }
8
+ const fields = action === 'render' ? ['chart', 'release', 'namespace'] : ['chart', 'release', 'namespace', 'context', 'fieldManager'];
9
+ fields.forEach((key) => {
10
+ if (typeof plan[key] !== 'string' || !plan[key].trim()) { throw new Error(`release requires ${key}`); }
11
+ });
12
+ }
13
+
14
+ /** Every Kubernetes command, including the post-rollout hook, shares this target. */
15
+ function kubeAt(plan, execute) {
16
+ return (args, input) => {
17
+ return execute('kubectl', ['--context', plan.context, '--namespace', plan.namespace, ...args],
18
+ { input, encoding: 'utf8', stdio: ['pipe', 'inherit', 'inherit'] });
19
+ };
20
+ }
21
+
22
+ function applyWith(kube, manifests, flags, checkOnly) {
23
+ const args = ['apply', ...flags, '-f', '-'];
24
+ kube([...args, '--dry-run=server'], manifests);
25
+ if (!checkOnly) { kube(args, manifests); }
26
+ }
27
+
28
+ /** Adopt legacy fields, then enforce the complete manifest before waiting for rollouts. */
29
+ function applyRelease(plan, action, kube, manifests) {
30
+ if (['adopt', 'check-adoption'].includes(action)) {
31
+ applyWith(kube, manifests, ['--field-manager=kubectl-client-side-apply'], action === 'check-adoption');
32
+ if (action === 'check-adoption') { return ''; }
33
+ }
34
+ applyWith(kube, manifests, ['--server-side', `--field-manager=${plan.fieldManager}`], action === 'check');
35
+ if (action === 'check') { return ''; }
36
+ for (const workload of plan.workloads || []) {
37
+ kube(['rollout', 'status', workload, '--timeout=300s']);
38
+ }
39
+ plan.afterReady?.(plan, kube);
40
+ return '';
41
+ }
42
+
43
+ /**
44
+ * Renders a Helm release and executes the selected kubectl action synchronously.
45
+ * Supply chart, release, namespace and values. Cluster actions additionally require
46
+ * an explicit context and fieldManager. prerequisites and workloads are kubectl
47
+ * resource/name references. afterReady(plan, kube) runs after a successful apply.
48
+ */
49
+ export default function executeRelease(plan, action = 'render', execute = execFileSync) {
50
+ validate(plan, action);
51
+ const manifests = execute('helm', ['template', plan.release, plan.chart, '--namespace', plan.namespace, '-f', '-'],
52
+ { input: JSON.stringify(plan.values), encoding: 'utf8' });
53
+ if (action === 'render') { return manifests; }
54
+ const kube = kubeAt(plan, execute);
55
+ for (const resource of plan.prerequisites || []) { kube(['get', resource, '-o', 'name']); }
56
+ if (action === 'diff') {
57
+ return kube(['diff', '--server-side', `--field-manager=${plan.fieldManager}`, '-f', '-'], manifests);
58
+ }
59
+ return applyRelease(plan, action, kube, manifests);
60
+ }
@@ -32,12 +32,19 @@ export default function segmentCodec(collector) {
32
32
  return {
33
33
  async accept(raw) {
34
34
  try {
35
- const parsed = JSON.parse(raw.payload);
35
+ let parsed;
36
+ let record;
37
+ try {
38
+ parsed = JSON.parse(raw.payload);
39
+ record = parsed.type === 'split-family' ? familyRecord(parsed) : segmentNormalize(parsed);
40
+ } catch (error) {
41
+ error.permanent = true;
42
+ throw error;
43
+ }
36
44
  if (parsed.type === 'split-family') {
37
- await collector.accept(familyRecord(parsed));
45
+ await collector.accept(record);
38
46
  return;
39
47
  }
40
- const record = segmentNormalize(parsed);
41
48
  await collector.accept(record);
42
49
  } catch (error) {
43
50
  processingErrorLog('segment_codec', error, { destination: raw.destination, payload: raw.payload });
@@ -60,6 +60,7 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
60
60
  const dispatch = segmentDispatch(continued, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
61
61
  const coalesce = segmentCoalesce(segmentCodec(dispatch), {
62
62
  size: config.size,
63
+ deadLetter: config.deadLetter,
63
64
  interval: config.interval
64
65
  });
65
66
  const destination = config.segmentsDestination || segmentsIngestDestination;
@@ -68,7 +69,8 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
68
69
  passcode: config.passcode,
69
70
  host: config.host,
70
71
  serial: false,
71
- manualAck: true
72
+ manualAck: true,
73
+ prefetch: config.prefetch
72
74
  });
73
75
  const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
74
76
  intervalMs: (config.poll || 5) * 1000
@@ -0,0 +1,24 @@
1
+ /** Builds durable topic exchanges, queues and AMQP 0-9-1 outboxes for one virtual host. */
2
+ export default function rabbitmqDefinitions({ vhost, users = [], permissions = [], exchanges = [],
3
+ queues = [], bindings = [], outboxes = [] }) {
4
+ if (!vhost) { throw new Error('a virtual host is required'); }
5
+ const routes = outboxes.flatMap(({ exchange, queue, keys }) => {
6
+ return keys.map((key) => { return { source: exchange, destination: queue, key }; });
7
+ });
8
+ const names = [...queues, ...outboxes.map(({ queue }) => { return queue; })];
9
+ return { vhosts: [{ name: vhost }], users, permissions,
10
+ exchanges: exchanges.map((name) => { return { name, vhost, type: 'topic', durable: true, auto_delete: false }; }),
11
+ queues: [...new Set(names)].map((name) => { return { name, vhost, durable: true, auto_delete: false, arguments: {} }; }),
12
+ bindings: [...bindings, ...routes].map(({ source, destination, key }) => {
13
+ return { source, vhost, destination, destination_type: 'queue', routing_key: key, arguments: {} };
14
+ }),
15
+ parameters: outboxes.filter(({ shovel }) => { return shovel; }).map(({ queue, shovel }) => {
16
+ return { vhost, component: 'shovel', name: shovel.name, value: {
17
+ 'src-protocol': 'amqp091', 'src-uri': shovel.sourceUri, 'src-queue': queue,
18
+ 'dest-protocol': 'amqp091', 'dest-uri': shovel.destinationUri,
19
+ 'dest-exchange': shovel.destinationExchange, 'reconnect-delay': shovel.reconnectDelay,
20
+ 'dest-publish-properties': { delivery_mode: 2 }
21
+ } };
22
+ })
23
+ };
24
+ }