@leverege/build-tools 2.65.1 → 2.66.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.65.1",
+  "version": "2.66.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -83,10 +83,10 @@
     "semver": "^7.7.1",
     "simple-git": "^3.27.0",
     "toml": "^3.0.0",
-    "zx": "^8.5.0"
+    "zx": "^8.5.2"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.0.1",
-    "npm": "^11.2.0"
+    "npm": "^11.3.0"
   }
 }

package/src/clone-cnpg-from-snapshot.mjs

@@ -83,7 +83,7 @@ const touchFile = async ( filename ) => {
 
 // generates an executable script to use for defining the volume on k8s
 const writeApplyScript = async ( dir ) => {
-  const applyScript = `#!/ust/bin/env bash
+  const applyScript = `#!/usr/bin/env bash
 #
 # Apply VolumeSnapshot resources prior to deploying the CNPG cluster
 #
@@ -387,7 +387,7 @@ const main = async () => {
   log( `
   ✅ Done. Snapshot: ${chalk.cyan( destSnapshot )}
            Recovery YAMLs written to: ${chalk.cyan( emitYamlDir )}
-           Run '${chalk.yellow( './apply-recovery-yaml.sh' )}' to prep the cluster for recovery.
+           Run '${chalk.yellow( './apply-volumesnapshots.sh' )}' to prep the cluster for recovery.
   ` )
 }
 
@@ -434,7 +434,7 @@ const cleanupResources = async () => {
     } ) )
   )
 
-  log( chalk.green( '  [✓] Parallel complete. Final GCE snapshot retained for CNPG bootstrapping.' ) )
+  log( chalk.green( '\n  [✓] Parallel cleanup complete - final GCE snapshot retained for CNPG bootstrapping\n' ) )
 }
 
 if ( cleanupOnly ) {
@@ -447,3 +447,5 @@ await main().catch( err => errorExit( err.message || err ) )
 if ( cleanup ) {
   await cleanupResources()
 }
+
+log( chalk.green( `  [✓] Cloning complete - recovery image name is => ${chalk.bold.yellow( k8sSnapshotName )}\n` ) )

package/src/helm-charts/cnpg-operator/cnpg-cleanup-cronjob.yaml.ovh

@@ -0,0 +1,67 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cnpg-cleanup-sa
+  namespace: cnpg-operands
+  annotations:
+    # wire up this KSA to the GSA to hook into IAM for workload identity
+    iam.gke.io/gcp-service-account: cnpg-cleanup-sa@${PROJECT_ID}.iam.gserviceaccount.com
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cnpg-cleanup-role
+  namespace: cnpg-operands
+rules:
+  - apiGroups: ["postgresql.cnpg.io"]
+    resources: ["backups"]
+    verbs: ["get", "list", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cnpg-cleanup-rolebinding
+  namespace: cnpg-operands
+subjects:
+  - kind: ServiceAccount
+    name: cnpg-cleanup-sa
+    namespace: cnpg-operands
+roleRef:
+  kind: Role
+  name: cnpg-cleanup-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cnpg-cleanup-snapshot
+  namespace: cnpg-operands
+spec:
+  schedule: "0 11 * * *" # 11 UTC is ~7AM EST
+# schedule: "*/5 * * * *" # every 5 minutes for testing
+  successfulJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          serviceAccountName: cnpg-cleanup-sa
+          containers:
+            - name: cnpg-cleanup
+              image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest
+              env:
+                - name: DRY_RUN
+                  value: "false"
+                - name: RETENTION_DAYS
+                  value: "21"
+              command:
+                - /bin/bash
+                - -c
+                - |
+                  echo "🚀 Fetching cleanup script from GCS..."
+                  apt-get update -qq
+                  apt-get install -y jq libjq1 libonig5 --no-install-recommends -o=Dpkg::Use-Pty=0 -o=APT::Get::Assume-Yes=1 -o=Debug::pkgProblemResolver=1
+                  gsutil cat gs://${PROJECT_ID}-k8s-cronjobs/cnpg-cleanup-script.sh | bash

package/src/helm-charts/cnpg-operator/cnpg-cleanup-script.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+set -euo pipefail
+
+KUBECTL_DELETE_ARGS=""
+[[ "$DRY_RUN" == "true" ]] && KUBECTL_DELETE_ARGS="--dry-run=server"
+
+# volumesnapshots namespace can be set here but if we need more than just the
+# cnpg-operands then we'll need to the Role with a ClusterRole and bind it
+# differently as well.
+VSNAPS_NS="-n cnpg-operands"
+
+# Default retention period (can override with env)
+RETENTION_DAYS="${RETENTION_DAYS:-21}"
+CURRENT_DATE=$(date +%s)
+
+log() {
+  echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*"
+}
+
+should_delete() {
+  local creation_ts="$1"
+  local age_days
+  local created_secs
+  created_secs=$(date -d "$creation_ts" +%s)
+  age_days=$(( (CURRENT_DATE - created_secs) / 86400 ))
+  if (( age_days > RETENTION_DAYS )); then
+    echo "$age_days"
+    return 0
+  fi
+  return 1
+}
+
+cleanup_cnpg_backups() {
+  log "🔍 Cleaning CNPG backups older than $RETENTION_DAYS days"
+  kubectl get backups.postgresql.cnpg.io -n cnpg-operands -o json | jq -c '.items[]' | while read -r item; do
+    name=$(echo "$item" | jq -r '.metadata.name')
+    created=$(echo "$item" | jq -r '.metadata.creationTimestamp')
+    [[ "$name" != *"-backup-"* ]] && log "Skipping manual backup: $name" && continue
+
+    if age=$(should_delete "$created"); then
+      log "🗑 Deleting CNPG backup: $name (Age: $age days)"
+      kubectl delete backups.postgresql.cnpg.io "$name" -n cnpg-operands --ignore-not-found --wait=false $KUBECTL_DELETE_ARGS
+    fi
+  done
+  log "✅ CNPG backup cleanup complete"
+}
+
+cleanup_volume_snapshots() {
+  log "🔍 Cleaning VolumeSnapshots older than $RETENTION_DAYS days"
+  kubectl get volumesnapshots $VSNAPS_NS -o json | jq -c '.items[]' | while read -r item; do
+    ns=$(echo "$item" | jq -r '.metadata.namespace')
+    name=$(echo "$item" | jq -r '.metadata.name')
+    created=$(echo "$item" | jq -r '.metadata.creationTimestamp')
+    ready=$(echo "$item" | jq -r '.status.readyToUse // false')
+
+    [[ "$ready" != "true" ]] && continue
+
+    if age=$(should_delete "$created"); then
+      log "🗑 Deleting VolumeSnapshot: $ns/$name (Age: $age days)"
+      kubectl delete volumesnapshot -n "$ns" "$name" --ignore-not-found --wait=false $KUBECTL_DELETE_ARGS
+    fi
+  done
+  log "✅ VolumeSnapshot cleanup complete"
+}
+
+main() {
+  log "🚀 Starting cleanup job (retention: $RETENTION_DAYS days)"
+  cleanup_cnpg_backups
+  cleanup_volume_snapshots
+  log "🏁 All cleanup tasks complete"
+}
+
+main

package/src/helm-charts/cnpg-operator/gitignore

@@ -0,0 +1 @@
+cnpg-cleanup-cronjob.yaml

package/src/helm-charts/cnpg-operator/helmup-1.26.0-rc1.plugin

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# See => https://cloudnative-pg.io/documentation/current/installation_upgrade/
+#
+# OPVER="1.25.1"
+OPVER="1.26.0-rc1" # 03/28/2025
+
+createNamespaceIfNeeded cnpg-system
+
+kubectl apply --server-side -f \
+    https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-${OPVER}.yaml $K8S_WHAT
+
+# Legacy terraformed clusters (like sandbox) may need a special firewall rule
+# added to the network layer on k8s. It should look like this:
+#
+# Name   : cnpg-operator
+# Ports  : 8000,9443     <= kubectl cnpg status and webhooks
+# Filters: 172.16.0.0/28 <= k8s control plane
+#
+# Old approach used the helm chart.
+#
+#addHelmRepo cnpg https://cloudnative-pg.github.io/charts
+#
+#helm upgrade --install cnpg cnpg/cloudnative-pg \
+#  --namespace cnpg-system --create-namespace \
+#  --set webhook.port="10250" $HELM_WHAT
+#
+#removeHelmRepo cnpg

package/src/helm-charts/cnpg-operator/helmup.plugin

@@ -2,13 +2,12 @@
 #
 # See => https://cloudnative-pg.io/documentation/current/installation_upgrade/
 #
-# OPVER="1.25.1"
-OPVER="1.26.0-rc1" # 03/28/2025
+OPVER="1.25.1"
 
 createNamespaceIfNeeded cnpg-system
 
 kubectl apply --server-side -f \
-    https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-${OPVER}.yaml $K8S_WHAT
+    https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${OPVER%.*}/releases/cnpg-${OPVER}.yaml $K8S_WHAT
 
 # Legacy terraformed clusters (like sandbox) may need a special firewall rule
 # added to the network layer on k8s. It should look like this:

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,7 +2,7 @@
 #
 showInstalling "Elastic Search 8"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.4.8"
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.5.0"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 

package/src/helm-charts/prom-operator/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "The Prometheus Operator and Components"
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
 showInstalling "The Prometheus Operator (kube-prometheus-stack)"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="69"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="70"
 
 NS="--namespace prometheus"
 

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -19,7 +19,7 @@ defaultRules:
     kubePrometheusGeneral: false
     kubePrometheusNodeRecording: false
     kubernetesApps: false
-    kubernetesResources: false
+    kubernetesResources: true
     kubernetesStorage: false
     kubernetesSystem: false
     kubeScheduler: false

package/src/helm-charts/prom-operator/rules/awesome-base/elasticsearch-rules.yaml

@@ -0,0 +1,219 @@
+# https://samber.github.io/awesome-prometheus-alerts/rules#elasticsearch
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-community-elasticsearch-exporter
+  namespace: prometheus
+spec:
+  groups:
+    - name: Elasticsearch (awesome) # PrometheusCommunityElasticsearchExporter-rules
+      rules:
+        - alert: ElasticsearchHeapUsageTooHigh
+          expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})
+            description: |-
+              The heap usage is over 90%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHeapUsageWarning
+          expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch Heap Usage warning (instance {{ $labels.instance }})
+            description: |-
+              The heap usage is over 80%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchDiskOutOfSpace
+          expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch disk out of space (instance {{ $labels.instance }})
+            description: |-
+              The disk usage is over 90%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchDiskSpaceLow
+          expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch disk space low (instance {{ $labels.instance }})
+            description: |-
+              The disk usage is over 80%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchClusterRed
+          expr: elasticsearch_cluster_health_status{color="red"} == 1
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Cluster Red (instance {{ $labels.instance }})
+            description: |-
+              Elastic Cluster Red status
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchClusterYellow
+          expr: elasticsearch_cluster_health_status{color="yellow"} == 1
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch Cluster Yellow (instance {{ $labels.instance }})
+            description: |-
+              Elastic Cluster Yellow status
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHealthyNodes
+          expr: elasticsearch_cluster_health_number_of_nodes < 3
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Healthy Nodes (instance {{ $labels.instance }})
+            description: |-
+              Missing node in Elasticsearch cluster
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHealthyDataNodes
+          expr: elasticsearch_cluster_health_number_of_data_nodes < 3
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch Healthy Data Nodes (instance {{ $labels.instance }})
+            description: |-
+              Missing data node in Elasticsearch cluster
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchRelocatingShards
+          expr: elasticsearch_cluster_health_relocating_shards > 0
+          for: 0m
+          labels:
+            severity: info
+          annotations:
+            summary: Elasticsearch relocating shards (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch is relocating shards
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchRelocatingShardsTooLong
+          expr: elasticsearch_cluster_health_relocating_shards > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch relocating shards too long (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has been relocating shards for 15min
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchInitializingShards
+          expr: elasticsearch_cluster_health_initializing_shards > 0
+          for: 0m
+          labels:
+            severity: info
+          annotations:
+            summary: Elasticsearch initializing shards (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch is initializing shards
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchInitializingShardsTooLong
+          expr: elasticsearch_cluster_health_initializing_shards > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch initializing shards too long (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has been initializing shards for 15 min
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchUnassignedShards
+          expr: elasticsearch_cluster_health_unassigned_shards > 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Elasticsearch unassigned shards (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has unassigned shards
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchPendingTasks
+          expr: elasticsearch_cluster_health_number_of_pending_tasks > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch pending tasks (instance {{ $labels.instance }})
+            description: |-
+              Elasticsearch has pending tasks. Cluster works slowly.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchNoNewDocuments
+          expr: increase(elasticsearch_indices_indexing_index_total{es_data_node="true"}[10m]) < 1
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch no new documents (instance {{ $labels.instance }})
+            description: |-
+              No new documents for 10 min!
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighIndexingLatency
+          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.0005
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Indexing Latency (instance {{ $labels.instance }})
+            description: |-
+              The indexing latency on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighIndexingRate
+          expr: sum(rate(elasticsearch_indices_indexing_index_total[1m]))> 10000
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Indexing Rate (instance {{ $labels.instance }})
+            description: |-
+              The indexing rate on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighQueryRate
+          expr: sum(rate(elasticsearch_indices_search_query_total[1m])) > 100
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Query Rate (instance {{ $labels.instance }})
+            description: |-
+              The query rate on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: ElasticsearchHighQueryLatency
+          expr: elasticsearch_indices_search_fetch_time_seconds / elasticsearch_indices_search_fetch_total > 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Elasticsearch High Query Latency (instance {{ $labels.instance }})
+            description: |-
+              The query latency on Elasticsearch cluster is higher than the threshold.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}

package/src/helm-charts/prom-operator/rules/awesome-base/etcd-exporter.yaml

@@ -0,0 +1,153 @@
+# https://samber.github.io/awesome-prometheus-alerts/rules#etcd
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: etcd-exporter # embedded-exporter
+  namespace: prometheus
+spec:
+  groups:
+    - name: Etcd (awesome) # EmbeddedExporter-rules
+      rules:
+        - alert: EtcdInsufficientMembers
+          expr: count(etcd_server_id) % 2 == 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Etcd insufficient Members (instance {{ $labels.instance }})
+            description: |-
+              Etcd cluster should have an odd number of members
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdNoLeader
+          expr: etcd_server_has_leader == 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Etcd no Leader (instance {{ $labels.instance }})
+            description: |-
+              Etcd cluster have no leader
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighNumberOfLeaderChanges
+          expr: increase(etcd_server_leader_changes_seen_total[10m]) > 2
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd high number of leader changes (instance {{ $labels.instance }})
+            description: |-
+              Etcd leader changed more than 2 times during 10 minutes
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighNumberOfFailedGrpcRequests
+          expr: sum(rate(grpc_server_handled_total{grpc_code!="OK"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.01
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd high number of failed GRPC requests (instance {{ $labels.instance }})
+            description: |-
+              More than 1% GRPC request failure detected in Etcd
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighNumberOfFailedGrpcRequests
+          expr: sum(rate(grpc_server_handled_total{grpc_code!="OK"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.05
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: Etcd high number of failed GRPC requests (instance {{ $labels.instance }})
+            description: |-
+              More than 5% GRPC request failure detected in Etcd
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdGrpcRequestsSlow
+          expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[1m])) by (grpc_service, grpc_method, le)) > 0.15
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd GRPC requests slow (instance {{ $labels.instance }})
+            description: |-
+              GRPC requests slowing down, 99th percentile is over 0.15s
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighNumberOfFailedHttpRequests
+          expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.01
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd high number of failed HTTP requests (instance {{ $labels.instance }})
+            description: |-
+              More than 1% HTTP failure detected in Etcd
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighNumberOfFailedHttpRequests
+          expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.05
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: Etcd high number of failed HTTP requests (instance {{ $labels.instance }})
+            description: |-
+              More than 5% HTTP failure detected in Etcd
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHttpRequestsSlow
+          expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[1m])) > 0.15
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd HTTP requests slow (instance {{ $labels.instance }})
+            description: |-
+              HTTP requests slowing down, 99th percentile is over 0.15s
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdMemberCommunicationSlow
+          expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) > 0.15
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd member communication slow (instance {{ $labels.instance }})
+            description: |-
+              Etcd member communication slowing down, 99th percentile is over 0.15s
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighNumberOfFailedProposals
+          expr: increase(etcd_server_proposals_failed_total[1h]) > 5
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd high number of failed proposals (instance {{ $labels.instance }})
+            description: |-
+              Etcd server got more than 5 failed proposals past hour
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighFsyncDurations
+          expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) > 0.5
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd high fsync durations (instance {{ $labels.instance }})
+            description: |-
+              Etcd WAL fsync duration increasing, 99th percentile is over 0.5s
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+        - alert: EtcdHighCommitDurations
+          expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) > 0.25
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Etcd high commit durations (instance {{ $labels.instance }})
+            description: |-
+              Etcd commit duration increasing, 99th percentile is over 0.25s
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}