@leverege/build-tools 2.66.3 → 2.66.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.66.3",
+  "version": "2.66.4",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/helm-charts/api-server/values-local.yaml

@@ -5,7 +5,7 @@
 #     - node tools/UpdateESMappingV2.js
 #
 image:
-  tag: # v5.17.5
+  tag: # v5.25.4
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/authz-server/values-local.yaml

@@ -4,7 +4,7 @@
 #     - npm run sequelizeRunMigrations
 #
 image:
-  tag: # v2.4.0
+  tag: # v2.5.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -18,7 +18,7 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
-        - CREATE DATABASE pg_stat_statements;
+        - CREATE EXTENSION pg_stat_statements;
         - CREATE DATABASE imagine;
 #       - CREATE DATABASE app; # for the operator restores
 #       - CREATE ROLE app;     # same

package/src/helm-charts/db-curator/values-local.yaml

@@ -1,7 +1,7 @@
 image:
-  tag: # v4.4.2
+  tag: # v4.6.2
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
   FIREBASE_CLEANUP_ENABLED: "true"
-  TSDB_CLEANUP_ENABLED: "false" # no need is new tsdb using autoexpiry
+  TSDB_CLEANUP_ENABLED: "false"

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,7 +2,8 @@
 #
 showInstalling "Elastic Search 8"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.6.2"
+# *** WARNING TO FUTURE SELF *** v22 of the chart is ES v9!!!
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.6.3"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 

package/src/helm-charts/fota-server/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v2.1.0
+  tag: # v2.1.3
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/message-processor/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v4.2.0
+  tag: # v4.3.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/messenger/values-local.yaml

@@ -1,5 +1,5 @@
 image:
- tag: # v2.14.0
+ tag: # v2.16.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/prom-operator/helmup.plugin

@@ -3,28 +3,35 @@
 showInstalling "The Prometheus Operator and Components"
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-showInstalling "The Prometheus Operator (kube-prometheus-stack)"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="70"
-
 NS="--namespace prometheus"
 
+showInstalling "The Prometheus Operator (kube-prometheus-stack)"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="72"
+
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 helm upgrade $NS --install prometheus-stack prometheus-community/kube-prometheus-stack \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-elasticsearch-exporter
-showInstalling "The Elasticsearch Exporter (prom-operator)"
+showInstalling "The Elasticsearch Exporter (prom-operator ES metrics exports)"
 [ -z "$ELASTICSEARCH_EXPORTER_CHART_VERSION" ] && ELASTICSEARCH_EXPORTER_CHART_VERSION="6"
 helm upgrade $NS --install elasticsearch8-exporter prometheus-community/prometheus-elasticsearch-exporter \
                  --values prom-operator/elasticsearch-exporter.yaml \
                  --version $ELASTICSEARCH_EXPORTER_CHART_VERSION $HELM_WHAT
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter
-showInstalling "The Stackdriver Exporter (prom-operator)"
+showInstalling "The Stackdriver Exporter (prom-operator GKE metrics exports)"
 [ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="4"
 helm upgrade $NS --install stackdriver-exporter prometheus-community/prometheus-stackdriver-exporter \
                  --values prom-operator/stackdriver-exporter.yaml \
                  --version $STACKDRIVER_EXPORTER_CHART_VERSION $HELM_WHAT
 
+# https://artifacthub.io/packages/helm/prometheus-community/prometheus-adapter
+showInstalling "The Prometheus Adapter (prom-operator custom HPA metrics)"
+[ -z "$PROMETHEUS_ADAPTER_CHART_VERSION" ] && PROMETHEUS_ADAPTER_CHART_VERSION="4"
+helm upgrade $NS --install prometheus-adapter prometheus-community/prometheus-adapter \
+                 --values prom-operator/prometheus-adapter.yaml \
+                 --version $PROMETHEUS_ADAPTER_CHART_VERSION $HELM_WHAT
+
 removeHelmRepo prometheus-community

package/src/helm-charts/prom-operator/prometheus-adapter.yaml

@@ -0,0 +1,18 @@
+fullnameOverride: "prometheus-stack-adapter"
+
+prometheus:
+  url: "http://prometheus-operated.prometheus.svc"
+  port: 9090
+
+rules:
+  custom:
+    - seriesQuery: 'gpu_duty_cycle_percent{namespace!="",pod!=""}'
+      resources:
+        overrides:
+          namespace: {resource: "namespace"}
+          pod: {resource: "pod"}
+      name:
+        matches: "gpu_duty_cycle_percent"
+        as: "gpu_duty_cycle_percent"
+      metricsQuery: |
+        avg(gpu_duty_cycle_percent{<<.LabelMatchers>>}) by (<<.GroupBy>>)

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -82,7 +82,7 @@ grafana:
         entryPoints:
           - websecure
         routes:
-          - match: Host(`${PROJECT_NAME}-promop.${HOST}.com`)
+          - match: Host(`${PROJECT_NAME}-monitoring.${HOST}.com`)
             kind: Rule
             services:
             - name: prometheus-stack-grafana
@@ -114,16 +114,57 @@ alertmanager:
         slack_configs:
           - channel: "${SLACK_CHANNEL}"
             api_url: "${SLACK_HOOK_URL}"
-            username: "${PROJECT_NAME}"
-            fallback: "${PROJECT_NAME} - {{ .CommonAnnotations.summary }}"
-            title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"
-            title_link: "https://bitbucket.org/leverege/sre-playbook/src/master/"
+            username: "${PROJECT_ID}"
             send_resolved: true
+            title: '{{ .CommonLabels.alertname }} alert'
+            title_link: '#'
+            color: >-
+              {{- if eq .Status "resolved" -}}
+                good
+              {{- else if eq .CommonLabels.severity "critical" -}}
+                danger
+              {{- else if eq .CommonLabels.severity "warning" -}}
+                warning
+              {{- else if eq .CommonLabels.severity "info" -}}
+                #439FE0
+              {{- else -}}
+                #CCCCCC
+              {{- end }}
             text: |-
               {{ range .Alerts }}
-                *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
-                *Description:* {{ .Annotations.description }}
-                *Dashboard:* {{ .Annotations.dashboard }}
+              {{- if eq .Status "resolved" }}
+                :white_check_mark: *Resolved:* {{ .Annotations.summary }}
+              {{- else if eq .Labels.severity "critical" }}
+                :fire: *Critical:* {{ .Annotations.summary }}
+              {{- else if eq .Labels.severity "warning" }}
+                :warning: *Warning:* {{ .Annotations.summary }}
+              {{- else if eq .Labels.severity "info" }}
+                :information_source: *Info:* {{ .Annotations.summary }}
+              {{- else }}
+                :grey_question: *Alert:* {{ .Annotations.summary }}
+              {{- end }}
+
+              *Description:* {{ .Annotations.description }}
+
+              {{- if .Annotations.runbook_url }}
+              *Runbook:* <{{ .Annotations.runbook_url }}|Open Playbook>
+              {{- end }}
+
+              {{- if .Labels.pod }}
+              *Pod Link:* *<https://console.cloud.google.com/kubernetes/pod/${GCE_REGION}/${CLUSTER_NAME}/{{ .Labels.namespace }}/{{ .Labels.pod }}?project=${PROJECT_ID}|`{{ .Labels.pod }}`>*
+              {{- end }}
+
+              *Status:* `{{ .Status }}` | *Severity:* `{{ .Labels.severity }}` {{- if .Labels.container }} *Container:* `{{ .Labels.container }}` {{- end }}
+  
+              {{- if .Annotations.dashboard }}
+              *Dashboard:* <{{ .Annotations.dashboard }}|View Dashboard>
+              {{- end }}
+
+              *Labels:*
+              {{- range $k, $v := .Labels }}
+                • `{{ $k }} = {{ $v }}`
+              {{- end }}
+              ---
               {{ end }}
       - name: slack-watchdog
         slack_configs:
@@ -156,10 +197,10 @@ alertmanager:
       group_interval: 5m
       group_by:
         - alertname
-        - cluster
+        - namespace
       receiver: slack
       routes:
-      - match:
+      - match_re:
           alertname: "Watchdog"
         receiver: slack-watchdog
       - match:

package/src/helm-charts/prom-operator/rules/cadvisor-rules.yaml

@@ -64,16 +64,31 @@ spec:
                 VALUE = {{ $value }}
                 LABELS = {{ $labels }}
         - alert: ContainerHighThrottleRate
-          expr: sum(increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) by (container, pod, namespace) / sum(increase(container_cpu_cfs_periods_total[5m])) by (container, pod, namespace) > ( 25 / 100 )
+          expr: |
+            (
+              (
+                sum by (container, pod, namespace) (increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m]))
+                /
+                sum by (container, pod, namespace) (increase(container_cpu_cfs_periods_total{container!=""}[5m]))
+              )
+              and on (pod, namespace)
+              (
+                time() - kube_pod_start_time > 60
+              )
+            ) > 0.3
           for: 5m
           labels:
             severity: warning
+            namespace: '{{ $labels.namespace }}'
+            pod: '{{ $labels.pod }}'
+            container: '{{ $labels.container }}'
           annotations:
-            summary: Container high throttle rate (instance {{ $labels.instance }})
-            description: |-
-              Container is being throttled
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+            summary: Pod experiencing high CPU throttling
+            description: |
+              {{ $labels.pod }} was throttled
+              - Verify Kubernetes CPU resource *requests* and *limits* for the container.
+              - Check if recent pod startup or autoscaling events could explain transient throttling.
+              - Persistent throttling indicates the container needs higher CPU limits or node sizing adjustment.
 #       - alert: ContainerHighLowChangeCpuUsage
 #         expr: (abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m] offset 1m)) * 100)) or abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[5m] offset 1m)) * 100))) > 25
 #         for: 0m

package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml

@@ -172,8 +172,8 @@ spec:
             summary: Kubernetes HPA scale inability (instance {{ $labels.instance }})
             description: |-
               HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to scale
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+#               VALUE = {{ $value }}
+#               LABELS = {{ $labels }}
         - alert: KubernetesHpaMetricsUnavailability
           expr: kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive"} == 1
           for: 0m
@@ -183,8 +183,8 @@ spec:
             summary: Kubernetes HPA metrics unavailability (instance {{ $labels.instance }})
             description: |-
               HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to collect metrics
-                VALUE = {{ $value }}
                 LABELS = {{ $labels }}
+#               VALUE = {{ $value }}
         - alert: KubernetesHpaScaleMaximum
           expr: (kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas) and (kube_horizontalpodautoscaler_spec_max_replicas > 1) and (kube_horizontalpodautoscaler_spec_min_replicas != kube_horizontalpodautoscaler_spec_max_replicas)
           for: 2m
@@ -193,9 +193,7 @@ spec:
           annotations:
             summary: Kubernetes HPA scale maximum (instance {{ $labels.instance }})
             description: |-
-              HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit maximum number of desired pods
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+              HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit max replica pods of {{ $value }}
         - alert: KubernetesHpaUnderutilized
           expr: max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3
           for: 0m

package/src/helm-charts/prom-operator/stackdriver-exporter-gpu-rules.yaml

@@ -0,0 +1,44 @@
+#
+# these custom rules exist to essentially boil down the GPU information provided
+# by the stackdriver exporter into a form that is a little more humand readable
+# in an attempt to keep the grafana / alert manager rules a little less "verbose"
+#
+# In a nutshell, we take metrics in the form of:
+#   stackdriver_k_8_s_container_kubernetes_io_container_accelerator_duty_cycle
+# and create boiled down metrics in the form of:
+#   gpu_duty_cycle_percent
+#
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-stack-kube-prom-stackdriver.gpu-rules
+  namespace: prometheus
+  labels:
+    release: prometheus-stack  # or whatever your Prometheus release name is
+spec:
+  groups:
+    - name: gpu.rules
+      interval: 1m
+      rules:
+        - record: "gpu_duty_cycle_percent"
+          expr: |
+            label_replace(
+              label_replace(
+                avg by (
+                  pod_name,
+                  namespace_name,
+                  container_name,
+                  accelerator_id,
+                  model,
+                  make
+                ) (
+                  stackdriver_k_8_s_container_kubernetes_io_container_accelerator_duty_cycle
+                ),
+                "namespace", "$1", "namespace_name", "(.*)"
+              ),
+              "pod", "$1", "pod_name", "(.*)"
+            )
+          labels:
+            source: "stackdriver"
+            metric_type: "derived"
+            subsystem: "gpu"

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -8,24 +8,27 @@ stackdriver:
   projectId: "${PROJECT_NAME}"
   metrics:
     typePrefixes:  >
-      pubsub.googleapis.com/subscription/oldest_unacked_message_age,
+      agent.googleapis.com/cpu/utilization,
+      agent.googleapis.com/memory/bytes_used,
+      container.googleapis.com/accelerator/duty_cycle,
+      container.googleapis.com/accelerator/memory_usage,
+      container.googleapis.com/accelerator/request_count,
+      firebasedatabase.googleapis.com/io/database_load,
+      firebasedatabase.googleapis.com/network/active_connections,
+      firebasedatabase.googleapis.com/network/sent_bytes_count,
+      kubernetes.io/container/accelerator/duty_cycle,
+      kubernetes.io/container/accelerator/memory_total,
+      kubernetes.io/container/accelerator/memory_used,
+      monitoring.googleapis.com/uptime_check/check_passed,
       pubsub.googleapis.com/subscription/num_undelivered_messages,
+      pubsub.googleapis.com/subscription/oldest_unacked_message_age,
       pubsub.googleapis.com/subscription/pull_ack_request_count,
       pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,
       pubsub.googleapis.com/topic/message_sizes,
       pubsub.googleapis.com/topic/send_request_count,
-      firebasedatabase.googleapis.com/io/database_load,
-      firebasedatabase.googleapis.com/network/sent_bytes_count,
-      firebasedatabase.googleapis.com/network/active_connections,
-      container.googleapis.com/accelerator/duty_cycle,
-      container.googleapis.com/accelerator/memory_usage,
-      container.googleapis.com/accelerator/request_count,
-      agent.googleapis.com/memory/bytes_used,
-      agent.googleapis.com/cpu/utilization,
+      quota.googleapis.com/allocation/usage,
       storage.googleapis.com/network/received_bytes_count,
-      storage.googleapis.com/network/sent_bytes_count,
-      monitoring.googleapis.com/uptime_check/check_passed,
-      quota.googleapis.com/allocation/usage
+      storage.googleapis.com/network/sent_bytes_count
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter?modal=values&path=serviceMonitor
 serviceMonitor:

package/src/helm-charts/pubsub-pulse/values-local.yaml

@@ -1,6 +1,5 @@
 image:
-  tag: # v1.6.3
-# registry: us-docker.pkg.dev/leverege-registry/leverege/images
+  tag: # v4.1.1
 
 config:
   pulseDelay: "1000"

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.13.0"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.13.4"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/values-local.yaml \

package/src/helm-charts/resource-server/values-local.yaml

@@ -4,7 +4,7 @@
 #     - npm run sequelizeRunMigrations
 #
 image:
-  tag: # v1.14.0
+  tag: # v1.14.1
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/rule-engine/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v2.6.0
+  tag: # v3.6.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/traefik/helmup.plugin

@@ -20,7 +20,7 @@ addHelmRepo traefik https://helm.traefik.io/traefik
 [ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="35"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
-             --values  traefik/values-local.yaml \
+             --values  traefik/values.yaml \
              --version $TRAEFIK_CHART_VERSION $HELM_WHAT
 
 removeHelmRepo traefik

package/src/helm-charts/transponder-bq/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-bq
 
 image:
-  tag: # v5.3.4
+  tag: # v5.5.1
 
 config:
   TRANSPONDER_RUN_MODE: "bigQueryJson"

package/src/helm-charts/transponder-dh/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-dh
 
 image:
-  tag: # v5.3.4
+  tag: # v5.5.1
 
 config:
   TRANSPONDER_RUN_MODE: "denseHistoryCombined"

package/src/helm-charts/transponder-rt/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-rt
 
 image:
-  tag: # v5.3.4
+  tag: # v5.5.1
 
 config:
   TRANSPONDER_RUN_MODE: "rt"

package/src/helm-charts/transponder-tsdb/values-local.yaml

@@ -7,7 +7,7 @@
 nameOverride: transponder-tsdb
 
 image:
-  tag: # v5.3.4
+  tag: # v5.5.1
 
 config:
   TRANSPONDER_RUN_MODE: "timescale"

package/src/helmup.sh

@@ -49,6 +49,7 @@ AUXILIARY=(
 
 # zombie-killer re-enabled 20240801
 SYSTEM=(
+  prom-operator
   elasticsearch8
   redis
   cnpg-operator
@@ -1049,7 +1050,7 @@ CATBACKUP
       "cnpg-operator")
         installCnpgOperatorEnvironment
         bootstrapLocalSetup $SERVICE
-        installCronCnpgCleanupSnapshot
+        installCnpgCleanupCronjob
         ;;
 
       "cnpg-cronjob")