@leverege/build-tools 2.66.0 → 2.66.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.66.0",
+  "version": "2.66.2",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -83,7 +83,7 @@
     "semver": "^7.7.1",
     "simple-git": "^3.27.0",
     "toml": "^3.0.0",
-    "zx": "^8.5.2"
+    "zx": "^8.5.3"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.0.1",

package/src/bash-funcs

@@ -11,6 +11,7 @@
 DEVNULL="/dev/null"
 [ ! -z "$BUILD_TOOLS_DEBUG" ] && DEVNULL="/dev/stdout" && printf "\n***BUILD_TOOLS_DEBUG DEVNULL=>$DEVNULL\n"
 
+GITTOP=`git rev-parse --show-toplevel`
 DTS="`date +\"%Y%m%d-%H%M%S\"`"
 
 # Simple ANSI color coded string generator.

package/src/helm-charts/cnpg-db-pgbench/cluster.yaml.ovh

@@ -14,6 +14,7 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
+        - CREATE EXTENSION pg_stat_statements;
       database: imagine
       owner: imagine
       secret:
@@ -38,6 +39,7 @@ spec:
     shared_preload_libraries:
       - timescaledb
       - jsquery
+      - pg_stat_statements
     parameters:
 #     max_worker_processes: "60"
       max_worker_processes: "8"

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -18,7 +18,8 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
-        - CREATE DATABASE imagine
+        - CREATE DATABASE pg_stat_statements;
+        - CREATE DATABASE imagine;
 #       - CREATE DATABASE app; # for the operator restores
 #       - CREATE ROLE app;     # same
 #     owner: imagine
@@ -52,6 +53,7 @@ spec:
     shared_preload_libraries:
       - timescaledb
       - jsquery
+      - pg_stat_statements
     parameters:
 #     max_connections: "100"
 #     max_worker_processes: "60"

package/src/helm-charts/cnpg-db-tsdb-dense/cluster.yaml.ovh

@@ -17,6 +17,7 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
+        - CREATE EXTENSION pg_stat_statements;
         - CREATE DATABASE imagine; # leverege tsdb db
 #     secret:
 #       name: cnpg-db-tsdb-dense-imagine-pw # imagine db pw
@@ -48,6 +49,7 @@ spec:
     shared_preload_libraries:
       - timescaledb
       - jsquery
+      - pg_stat_statements
     parameters:
 #     max_worker_processes: "60"
       max_worker_processes: "8"

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,7 +2,7 @@
 #
 showInstalling "Elastic Search 8"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.5.0"
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.6.0"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -10,7 +10,7 @@ defaultRules:
     etcd: false
     configReloaders: false
     general: true
-    k8s: true
+    k8s: false # GPT see po-k8s-selective
     kubeApiserver: true  # GPT Keep an eye on API server availability
     kubeApiserverAvailability: false
     kubeApiserverSlos: false
@@ -19,7 +19,7 @@ defaultRules:
     kubePrometheusGeneral: false
     kubePrometheusNodeRecording: false
     kubernetesApps: false
-    kubernetesResources: false
+    kubernetesResources: true
     kubernetesStorage: false
     kubernetesSystem: false
     kubeScheduler: false

package/src/helm-charts/prom-operator/rules/awesome-base/elasticsearch-rules.yaml

@@ -0,0 +1,219 @@
+# https://samber.github.io/awesome-prometheus-alerts/rules#elasticsearch
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-community-elasticsearch-exporter
+  namespace: prometheus
+spec:
+  groups:
+    - name: Elasticsearch (awesome) # PrometheusCommunityElasticsearchExporter-rules
+      rules:
+        - alert: ElasticsearchHeapUsageTooHigh
+          expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})
+            description: |-
+              The heap usage is over 90%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHeapUsageWarning
+          expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch Heap Usage warning (instance {{ $labels.instance }})
+            description: |-
+              The heap usage is over 80%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchDiskOutOfSpace
+          expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch disk out of space (instance {{ $labels.instance }})
+            description: |-
+              The disk usage is over 90%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchDiskSpaceLow
+          expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch disk space low (instance {{ $labels.instance }})
+            description: |-
+              The disk usage is over 80%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchClusterRed
+          expr: elasticsearch_cluster_health_status{color="red"} == 1
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Cluster Red (instance {{ $labels.instance }})
+            description: |-
+              Elastic Cluster Red status
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchClusterYellow
+          expr: elasticsearch_cluster_health_status{color="yellow"} == 1
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch Cluster Yellow (instance {{ $labels.instance }})
+            description: |-
+              Elastic Cluster Yellow status
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHealthyNodes
+          expr: elasticsearch_cluster_health_number_of_nodes < 3
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Healthy Nodes (instance {{ $labels.instance }})
+            description: |-
+              Missing node in Elasticsearch cluster
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHealthyDataNodes
+          expr: elasticsearch_cluster_health_number_of_data_nodes < 3
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Healthy Data Nodes (instance {{ $labels.instance }})
+            description: |-
+              Missing data node in Elasticsearch cluster
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchRelocatingShards
+          expr: elasticsearch_cluster_health_relocating_shards > 0
+          for: 0m
+          labels:
+            severity: info
+          annotations:
+            summary: Elasticsearch relocating shards (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch is relocating shards
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchRelocatingShardsTooLong
+          expr: elasticsearch_cluster_health_relocating_shards > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch relocating shards too long (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has been relocating shards for 15min
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchInitializingShards
+          expr: elasticsearch_cluster_health_initializing_shards > 0
+          for: 0m
+          labels:
+            severity: info
+          annotations:
+            summary: Elasticsearch initializing shards (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch is initializing shards
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchInitializingShardsTooLong
+          expr: elasticsearch_cluster_health_initializing_shards > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch initializing shards too long (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has been initializing shards for 15 min
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchUnassignedShards
+          expr: elasticsearch_cluster_health_unassigned_shards > 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch unassigned shards (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has unassigned shards
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchPendingTasks
+          expr: elasticsearch_cluster_health_number_of_pending_tasks > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch pending tasks (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has pending tasks. Cluster works slowly.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchNoNewDocuments
+          expr: increase(elasticsearch_indices_indexing_index_total{es_data_node="true"}[10m]) < 1
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch no new documents (instance {{ $labels.instance }})
+            description: |-
+              No new documents for 10 min!
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighIndexingLatency
+          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.0005
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Indexing Latency (instance {{ $labels.instance }})
+            description: |-
+              The indexing latency on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighIndexingRate
+          expr: sum(rate(elasticsearch_indices_indexing_index_total[1m]))> 10000
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Indexing Rate (instance {{ $labels.instance }})
+            description: |-
+              The indexing rate on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighQueryRate
+          expr: sum(rate(elasticsearch_indices_search_query_total[1m])) > 100
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Query Rate (instance {{ $labels.instance }})
+            description: |-
+              The query rate on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighQueryLatency
+          expr: elasticsearch_indices_search_fetch_time_seconds / elasticsearch_indices_search_fetch_total > 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Query Latency (instance {{ $labels.instance }})
+            description: |-
+              The query latency on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}

package/src/helm-charts/prom-operator/rules/awesome-base/google-cadvisor.yaml

@@ -0,0 +1,108 @@
+# https://samber.github.io/awesome-prometheus-alerts/rules#docker-containers
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: google-cadvisor
+spec:
+  groups:
+    - name: Google Cadvisor (awesome) # GoogleCadvisor-rules
+      rules:
+        - alert: ContainerKilled
+          expr: time() - container_last_seen > 60
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: Container killed (instance {{ $labels.instance }})
+            description: |-
+              A container has disappeared
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerAbsent
+          expr: absent(container_last_seen)
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Container absent (instance {{ $labels.instance }})
+            description: |-
+              A container is absent for 5 min
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerHighCpuUtilization
+          expr: (sum(rate(container_cpu_usage_seconds_total{container!=""}[5m])) by (pod, container) / sum(container_spec_cpu_quota{container!=""}/container_spec_cpu_period{container!=""}) by (pod, container) * 100) > 80
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Container High CPU utilization (instance {{ $labels.instance }})
+            description: |-
+              Container CPU utilization is above 80%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerHighMemoryUsage
+          expr: (sum(container_memory_working_set_bytes{name!=""}) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Container High Memory usage (instance {{ $labels.instance }})
+            description: |-
+              Container Memory usage is above 80%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerVolumeUsage
+          expr: (1 - (sum(container_fs_inodes_free{name!=""}) BY (instance) / sum(container_fs_inodes_total) BY (instance))) * 100 > 80
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Container Volume usage (instance {{ $labels.instance }})
+            description: |-
+              Container Volume usage is above 80%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerHighThrottleRate
+          expr: sum(increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) by (container, pod, namespace) / sum(increase(container_cpu_cfs_periods_total[5m])) by (container, pod, namespace) > ( 25 / 100 )
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Container high throttle rate (instance {{ $labels.instance }})
+            description: |-
+              Container is being throttled
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerHighLowChangeCpuUsage
+          expr: (abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m] offset 1m)) * 100)) or abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[5m] offset 1m)) * 100))) > 25
+          for: 0m
+          labels:
+            severity: info
+          annotations:
+            summary: Container high low change CPU usage (instance {{ $labels.instance }})
+            description: |-
+              This alert rule monitors the absolute change in CPU usage within a time window and triggers an alert when the change exceeds 25%.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerLowCpuUtilization
+          expr: (sum(rate(container_cpu_usage_seconds_total{container!=""}[5m])) by (pod, container) / sum(container_spec_cpu_quota{container!=""}/container_spec_cpu_period{container!=""}) by (pod, container) * 100) < 20
+          for: 7d
+          labels:
+            severity: info
+          annotations:
+            summary: Container Low CPU utilization (instance {{ $labels.instance }})
+            description: |-
+              Container CPU utilization is under 20% for 1 week. Consider reducing the allocated CPU.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ContainerLowMemoryUsage
+          expr: (sum(container_memory_working_set_bytes{name!=""}) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) < 20
+          for: 7d
+          labels:
+            severity: info
+          annotations:
+            summary: Container Low Memory usage (instance {{ $labels.instance }})
+            description: |-
+              Container Memory usage is under 20% for 1 week. Consider reducing the allocated memory.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}

package/src/helm-charts/prom-operator/rules/{new-awesome.yaml → awesome-base/kubestate-exporter.yaml}

@@ -1,10 +1,11 @@
+# https://samber.github.io/awesome-prometheus-alerts/rules#kubernetes
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
   name: kubestate-exporter
 spec:
   groups:
-    - name: KubestateExporter-rules
+    - name: Kubestate (awesome) # KubestateExporter-rules
       rules:
         - alert: KubernetesNodeNotReady
           expr: kube_node_status_condition{condition="Ready",status="true"} == 0