@yarkivaev/scada 2.3.112 → 2.3.114

package/README.md

@@ -274,3 +274,15 @@ Depend on a pinned version (`"@yarkivaev/scada": "…#v2.3.46"`), not a local pa
 ## CI/CD
 
 GitHub Actions via `yarkivaev/npm-workflows` (lint, test, version check). This repo publishes an npm package only; container images are built in downstream deploy repos.
+
+### Ingest delivery and quarantine
+
+`STOMP_PREFETCH` sets a positive, bounded segment delivery window (default 128).
+`SEGMENTS_DEAD_LETTER_ENABLED=true` opts into rejecting malformed segment records
+without requeue; transient persistence failures are retried. Enable this only with
+a durable DLQ and routing policy. In the deployment DSL, configure
+`profiles.<profile>.modules.message_bus.dead_letter.queue: scada.ingest.dlq`.
+The scada chart declares the queue and applies an ingest-only dead-letter policy,
+without changing existing queue arguments. The profile `env` is passed to both
+site-server and supervisor-sink. DLQ records retain the original body and RabbitMQ
+`x-death` metadata for inspection and deliberate replay; they are not auto-expired.

package/charts/scada/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: scada
 description: Generic SCADA site — site-server, supervisor-sink, collectors, Postgres, RabbitMQ, optional ClickHouse
 type: application
-version: 0.2.1
-appVersion: "2.3.112"
+version: 0.2.4
+appVersion: "2.3.114"

package/charts/scada/templates/_helpers.tpl

@@ -2,6 +2,31 @@
 app.kubernetes.io/name: scada
 app.kubernetes.io/instance: {{ .Release.Name }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- include "scada.extraLabels" . }}
+{{- end }}
+
+{{- define "scada.extraLabels" -}}
+{{- range $key, $val := (.Values.extraLabels | default dict) }}
+{{ $key }}: {{ $val | quote }}
+{{- end }}
+{{- end }}
+
+{{- define "scada.match" -}}
+app: {{ .app }}
+{{- range $key, $val := (.ctx.Values.extraLabels | default dict) }}
+{{ $key }}: {{ $val | quote }}
+{{- end }}
+{{- end }}
+
+{{- define "scada.claim" -}}
+accessModes: ["ReadWriteOnce"]
+{{- if .class }}
+storageClassName: {{ .class | quote }}
+{{- end }}
+resources:
+  requests:
+    storage: {{ .size | quote }}
+volumeMode: Filesystem
 {{- end }}
 
 {{- define "scada.device" -}}

package/charts/scada/templates/clickhouse.yaml

@@ -12,9 +12,14 @@ spec:
     - name: http
       port: 8123
       targetPort: http
+    - name: native
+      port: 9000
+      targetPort: native
+    - name: metrics
+      port: 9363
+      targetPort: metrics
   selector:
-    app: clickhouse
-    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- include "scada.match" (dict "app" "clickhouse" "ctx" .) | nindent 4 }}
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -29,13 +34,12 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app: clickhouse
-      app.kubernetes.io/instance: {{ .Release.Name }}
+      {{- include "scada.match" (dict "app" "clickhouse" "ctx" .) | nindent 6 }}
   template:
     metadata:
       labels:
         {{- include "scada.labels" . | nindent 8 }}
-        app: clickhouse
+        {{- include "scada.match" (dict "app" "clickhouse" "ctx" .) | nindent 8 }}
     spec:
       containers:
         - name: clickhouse
@@ -43,6 +47,10 @@ spec:
           ports:
             - name: http
               containerPort: 8123
+            - name: native
+              containerPort: 9000
+            - name: metrics
+              containerPort: 9363
           volumeMounts:
             - name: data
               mountPath: /var/lib/clickhouse
@@ -50,8 +58,5 @@ spec:
     - metadata:
         name: data
       spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 20Gi
+        {{- include "scada.claim" (dict "size" .Values.clickhouse.storage "class" .Values.clickhouse.storageClass) | nindent 8 }}
 {{- end }}

package/charts/scada/templates/collector.yaml

@@ -62,14 +62,13 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app: collector
-      app.kubernetes.io/instance: {{ $.Release.Name }}
+      {{- include "scada.match" (dict "app" "collector" "ctx" $) | nindent 6 }}
       scada/command: {{ $slug }}
   template:
     metadata:
       labels:
         {{- include "scada.labels" $ | nindent 8 }}
-        app: collector
+        {{- include "scada.match" (dict "app" "collector" "ctx" $) | nindent 8 }}
         scada/command: {{ $slug }}
     spec:
       {{- with $.Values.pullSecret }}

package/charts/scada/templates/postgres.yaml

@@ -13,8 +13,7 @@ spec:
       port: 5432
       targetPort: postgres
   selector:
-    app: postgres
-    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- include "scada.match" (dict "app" "postgres" "ctx" .) | nindent 4 }}
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -29,13 +28,12 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app: postgres
-      app.kubernetes.io/instance: {{ .Release.Name }}
+      {{- include "scada.match" (dict "app" "postgres" "ctx" .) | nindent 6 }}
   template:
     metadata:
       labels:
         {{- include "scada.labels" . | nindent 8 }}
-        app: postgres
+        {{- include "scada.match" (dict "app" "postgres" "ctx" .) | nindent 8 }}
     spec:
       containers:
         - name: postgres
@@ -56,6 +54,8 @@ spec:
                 secretKeyRef:
                   name: {{ .Values.postgres.secret }}
                   key: password
+            - name: PGDATA
+              value: {{ .Values.postgres.data | quote }}
           volumeMounts:
             - name: data
               mountPath: /var/lib/postgresql/data
@@ -63,8 +63,5 @@ spec:
     - metadata:
         name: data
       spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 10Gi
+        {{- include "scada.claim" (dict "size" .Values.postgres.storage "class" .Values.postgres.storageClass) | nindent 8 }}
 {{- end }}

package/charts/scada/templates/rabbitmq.yaml

@@ -35,6 +35,11 @@
 {{- if $outgoing }}
 {{- $queue = append $queue (dict "name" "scada.telemetry.outbox" "vhost" "/" "durable" true "auto_delete" false) -}}
 {{- end }}
+{{- $policies := list -}}
+{{- with .Values.rabbitmq.deadLetter }}
+{{- $queue = append $queue (dict "name" .queue "vhost" "/" "durable" true "auto_delete" false) -}}
+{{- $policies = append $policies (dict "name" "ingest-dlq" "vhost" "/" "pattern" "^scada\\.(segments|operations|telemetry)\\.ingest$" "apply-to" "queues" "priority" 10 "definition" (dict "dead-letter-exchange" "" "dead-letter-routing-key" .queue)) -}}
+{{- end }}
 {{- $binding := list -}}
 {{- range $key := $ingest }}
 {{- $binding = append $binding (dict "source" "scada.telemetry" "vhost" "/" "destination" "scada.telemetry.ingest" "destination_type" "queue" "routing_key" $key) -}}
@@ -54,6 +59,7 @@
   "permissions" (list (dict "user" "guest" "vhost" "/" "configure" ".*" "write" ".*" "read" ".*"))
   "exchanges" $exchange
   "queues" $queue
+  "policies" $policies
   "bindings" $binding
 -}}
 apiVersion: v1
@@ -85,19 +91,38 @@ metadata:
     {{- include "scada.labels" . | nindent 4 }}
     app: rabbitmq
 spec:
+  {{- $svc := .Values.rabbitmq.service | default dict }}
+  type: {{ $svc.type | default "ClusterIP" }}
   ports:
     - name: amqp
       port: 5672
       targetPort: amqp
+      {{- with $svc.nodePorts }}
+      {{- if .amqp }}
+      nodePort: {{ .amqp }}
+      {{- end }}
+      {{- end }}
     - name: mqtt
       port: 1883
       targetPort: mqtt
     - name: stomp
       port: 61613
       targetPort: stomp
+      {{- with $svc.nodePorts }}
+      {{- if .stomp }}
+      nodePort: {{ .stomp }}
+      {{- end }}
+      {{- end }}
+    - name: management
+      port: 15672
+      targetPort: management
+      {{- with $svc.nodePorts }}
+      {{- if .management }}
+      nodePort: {{ .management }}
+      {{- end }}
+      {{- end }}
   selector:
-    app: rabbitmq
-    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- include "scada.match" (dict "app" "rabbitmq" "ctx" .) | nindent 4 }}
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -112,13 +137,12 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app: rabbitmq
-      app.kubernetes.io/instance: {{ .Release.Name }}
+      {{- include "scada.match" (dict "app" "rabbitmq" "ctx" .) | nindent 6 }}
   template:
     metadata:
       labels:
         {{- include "scada.labels" . | nindent 8 }}
-        app: rabbitmq
+        {{- include "scada.match" (dict "app" "rabbitmq" "ctx" .) | nindent 8 }}
     spec:
       containers:
         - name: rabbitmq
@@ -130,9 +154,18 @@ spec:
               containerPort: 1883
             - name: stomp
               containerPort: 61613
+            - name: management
+              containerPort: 15672
           volumeMounts:
             - name: config
-              mountPath: /etc/rabbitmq
+              mountPath: /etc/rabbitmq/enabled_plugins
+              subPath: enabled_plugins
+            - name: config
+              mountPath: /etc/rabbitmq/rabbitmq.conf
+              subPath: rabbitmq.conf
+            - name: config
+              mountPath: /etc/rabbitmq/definitions.json
+              subPath: definitions.json
             - name: data
               mountPath: /var/lib/rabbitmq
       volumes:
@@ -143,8 +176,5 @@ spec:
     - metadata:
         name: data
       spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 5Gi
+        {{- include "scada.claim" (dict "size" .Values.rabbitmq.storage "class" .Values.rabbitmq.storageClass) | nindent 8 }}
 {{- end }}

package/charts/scada/templates/site-server.yaml

@@ -10,13 +10,12 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app: site-server
-      app.kubernetes.io/instance: {{ .Release.Name }}
+      {{- include "scada.match" (dict "app" "site-server" "ctx" .) | nindent 6 }}
   template:
     metadata:
       labels:
         {{- include "scada.labels" . | nindent 8 }}
-        app: site-server
+        {{- include "scada.match" (dict "app" "site-server" "ctx" .) | nindent 8 }}
     spec:
       {{- with .Values.pullSecret }}
       imagePullSecrets:
@@ -85,8 +84,7 @@ spec:
       port: {{ .Values.service.internalPort }}
       targetPort: internal-api
   selector:
-    app: site-server
-    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- include "scada.match" (dict "app" "site-server" "ctx" .) | nindent 4 }}
 {{- if gt (int .Values.service.nodePort) 0 }}
 ---
 apiVersion: v1
@@ -105,6 +103,5 @@ spec:
       targetPort: http
       nodePort: {{ .Values.service.nodePort }}
   selector:
-    app: site-server
-    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- include "scada.match" (dict "app" "site-server" "ctx" .) | nindent 4 }}
 {{- end }}

package/charts/scada/templates/supervisor-sink.yaml

@@ -10,13 +10,12 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app: supervisor-sink
-      app.kubernetes.io/instance: {{ .Release.Name }}
+      {{- include "scada.match" (dict "app" "supervisor-sink" "ctx" .) | nindent 6 }}
   template:
     metadata:
       labels:
         {{- include "scada.labels" . | nindent 8 }}
-        app: supervisor-sink
+        {{- include "scada.match" (dict "app" "supervisor-sink" "ctx" .) | nindent 8 }}
     spec:
       {{- with .Values.pullSecret }}
       imagePullSecrets:
@@ -33,6 +32,10 @@ spec:
             - name: STATE_HTTP_PORT
               value: {{ .Values.service.internalPort | quote }}
             {{- include "scada.linkEnv" . | nindent 12 }}
+            {{- range .Values.env }}
+            - name: {{ .name }}
+              value: {{ .value | quote }}
+            {{- end }}
           readinessProbe:
             tcpSocket:
               port: state
@@ -60,5 +63,4 @@ spec:
       port: {{ .Values.service.internalPort }}
       targetPort: state
   selector:
-    app: supervisor-sink
-    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- include "scada.match" (dict "app" "supervisor-sink" "ctx" .) | nindent 4 }}

package/charts/scada/values.yaml

@@ -6,6 +6,7 @@ machines: []
 sites: []
 collectors: []
 env: []
+extraLabels: {}
 service:
   port: 3000
   internalPort: 8081
@@ -15,12 +16,22 @@ postgres:
   image: postgres:16
   secret: scada-postgres
   database: scada
+  storage: 10Gi
+  storageClass: ""
+  data: /var/lib/postgresql/data/pgdata
 rabbitmq:
   enabled: true
+  deadLetter: null
   image: rabbitmq:3.13-management
+  storage: 5Gi
+  storageClass: ""
+  service:
+    type: ClusterIP
 clickhouse:
   enabled: false
   image: clickhouse/clickhouse-server:24.8
+  storage: 20Gi
+  storageClass: ""
 resources:
   requests:
     cpu: 100m

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.112",
+  "version": "2.3.114",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",
@@ -31,7 +31,7 @@
     "@clickhouse/client": "^1.0.0",
     "@modelcontextprotocol/sdk": "^1.24.3",
     "@yarkivaev/simple-server": "1.0.2",
-    "@yarkivaev/source-to-sink": "1.3.8",
+    "@yarkivaev/source-to-sink": "1.3.10",
     "amqplib": "^0.10.9",
     "pg": "^8.0.0",
     "stompit": "^1.0.0",

package/src/application/deploy/releaseValues.js

@@ -16,7 +16,8 @@ function bindOne(name, spec, secret) {
         return ['postgres', { enabled: spec.enabled !== false, secret: secret || 'scada-postgres' }];
     }
     if (name === 'message_bus') {
-        return ['rabbitmq', { enabled: spec.enabled !== false }];
+        return ['rabbitmq', { enabled: spec.enabled !== false,
+            ...(spec.dead_letter ? { deadLetter: spec.dead_letter } : {}) }];
     }
     if (name === 'telemetry_store') {
         return ['clickhouse', { enabled: spec.enabled !== false }];

package/src/application/supervisorSink.js

@@ -23,6 +23,10 @@ const appRoot = path.dirname(fileURLToPath(import.meta.url));
  */
 export function readDeploymentConfig(env) {
     const root = path.join(appRoot, '../..');
+    const prefetch = Number(env.STOMP_PREFETCH || '128');
+    if (!Number.isSafeInteger(prefetch) || prefetch < 1) {
+        throw new Error('STOMP_PREFETCH must be a positive safe integer');
+    }
     return {
         stomp: env.STOMP_URL,
         postgres: env.POSTGRES_URL,
@@ -39,6 +43,8 @@ export function readDeploymentConfig(env) {
         requestTimeoutMs: parseRequestTimeoutMs(env.REQUEST_TIMEOUT_MS),
         pipelines: env.SINK_PIPELINES !== 'false',
         pipeline: {
+            prefetch,
+            deadLetter: env.SEGMENTS_DEAD_LETTER_ENABLED === 'true',
             size: parseInt(env.BATCH_SIZE || '50', 10),
             interval: parseInt(env.FLUSH_INTERVAL || '2', 10),
             threshold: parseInt(env.CIRCUIT_THRESHOLD || '5', 10),

package/src/domain/segment/segmentCoalesce.js

@@ -40,7 +40,7 @@ function isHeartbeat(parsed) {
  * @param {object} codec - downstream codec
  * @returns {Promise<void>}
  */
-async function flushLane(lane, codec) {
+async function flushLane(lane, codec, deadLetter) {
     const batch = lane.pending;
     if (!batch) {
         return;
@@ -57,9 +57,11 @@ async function flushLane(lane, codec) {
         }
     } catch (error) {
         for (const settle of batch.settles) {
-            settle.nack();
+            settle.nack(!(deadLetter && error.permanent));
+        }
+        if (!error.permanent) {
+            throw error;
         }
-        throw error;
     }
 }
 
@@ -78,7 +80,12 @@ async function bufferHeartbeat(ctx, envelope, parsed) {
         lane.pending.payload = envelope.payload;
         lane.pending.settles.push(envelope.settle);
     } else {
-        await api.flush(lane);
+        try {
+            await api.flush(lane);
+        } catch (error) {
+            envelope.settle.nack(true);
+            throw error;
+        }
         const pending = {
             key,
             destination: envelope.destination,
@@ -88,7 +95,7 @@ async function bufferHeartbeat(ctx, envelope, parsed) {
         const timer = setTimeout(() => {
             lane.queue = lane.queue.then(() => {
                 return api.flush(lane);
-            });
+            }).catch(() => { return undefined; });
         }, intervalMs);
         Object.assign(lane, { pending, timer });
     }
@@ -110,12 +117,12 @@ async function step(ctx, envelope, parsed) {
         await bufferHeartbeat(ctx, envelope, parsed);
         return;
     }
-    await ctx.api.flush(ctx.lane);
     try {
+        await ctx.api.flush(ctx.lane);
         await ctx.codec.accept({ destination: envelope.destination, payload: envelope.payload });
         envelope.settle.ack();
     } catch (error) {
-        envelope.settle.nack();
+        envelope.settle.nack(!(ctx.deadLetter && error.permanent));
         throw error;
     }
 }
@@ -148,9 +155,18 @@ export default function segmentCoalesce(codec, options = {}) {
     const lanes = new Map();
     const api = {
         accept(envelope) {
-            const parsed = JSON.parse(envelope.payload);
+            let parsed;
+            try {
+                parsed = JSON.parse(envelope.payload);
+                if (!parsed || typeof parsed !== 'object') {
+                    throw new Error('Segment must be an object');
+                }
+            } catch {
+                envelope.settle.nack(!options.deadLetter);
+                return Promise.resolve();
+            }
             const lane = laneState(lanes, parsed.machine);
-            const ctx = { api, lane, codec, size, intervalMs };
+            const ctx = { api, lane, codec, size, intervalMs, deadLetter: options.deadLetter };
             lane.queue = lane.queue.then(() => {
                 return step(ctx, envelope, parsed);
             }).catch(() => {
@@ -159,7 +175,7 @@ export default function segmentCoalesce(codec, options = {}) {
             return lane.queue;
         },
         async flush(lane) {
-            await flushLane(lane, codec);
+            await flushLane(lane, codec, options.deadLetter);
         }
     };
     return api;

package/src/infrastructure/ingest/codecs/segmentCodec.js

@@ -32,12 +32,19 @@ export default function segmentCodec(collector) {
     return {
         async accept(raw) {
             try {
-                const parsed = JSON.parse(raw.payload);
+                let parsed;
+                let record;
+                try {
+                    parsed = JSON.parse(raw.payload);
+                    record = parsed.type === 'split-family' ? familyRecord(parsed) : segmentNormalize(parsed);
+                } catch (error) {
+                    error.permanent = true;
+                    throw error;
+                }
                 if (parsed.type === 'split-family') {
-                    await collector.accept(familyRecord(parsed));
+                    await collector.accept(record);
                     return;
                 }
-                const record = segmentNormalize(parsed);
                 await collector.accept(record);
             } catch (error) {
                 processingErrorLog('segment_codec', error, { destination: raw.destination, payload: raw.payload });

package/src/infrastructure/ingest/pipelines/segmentPipeline.js

@@ -60,6 +60,7 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
     const dispatch = segmentDispatch(continued, retagSink(pool), splitSink, closeOrphanOpen(pool), pool);
     const coalesce = segmentCoalesce(segmentCodec(dispatch), {
         size: config.size,
+        deadLetter: config.deadLetter,
         interval: config.interval
     });
     const destination = config.segmentsDestination || segmentsIngestDestination;
@@ -68,7 +69,8 @@ export default function segmentPipeline(stomp, postgres, pool, config) {
         passcode: config.passcode,
         host: config.host,
         serial: false,
-        manualAck: true
+        manualAck: true,
+        prefetch: config.prefetch
     });
     const silence = silentOpenWatch(closeSilentOpen(pool), budget, {
         intervalMs: (config.poll || 5) * 1000