@leverege/build-tools 2.66.0 → 2.66.1

package/src/helm-charts/prom-operator/rules/traefik-rules.yaml

@@ -1,27 +1,54 @@
+# https://samber.github.io/awesome-prometheus-alerts/rules#traefik
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: traefik-rules
+  name: embedded-exporter-v2
   namespace: prometheus
-
 spec:
   groups:
-  - name: Traefik Alerts
-    rules:
-    - alert: TraefikConfigReloadFailed
-      expr: traefik_config_last_reload_failure > 0
-      for: 1m
-      labels:
-        severity: warning
-      annotations:
-        summary: "Traefik config failed to reload"
-        description: "New config file for Traefik is invalid or Traefik encountered an error"
-
-#   - alert: TraefikHighHttp4xxError
-#     expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
-#     for: 5m
-#     labels:
-#       severity: error
-#     annotations:
-#       summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
-#       description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
+    - name: EmbeddedExporterV2-rules
+      rules:
+        - alert: TraefikServiceDown
+          expr: count(traefik_service_server_up) by (service) == 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Traefik service down (instance {{ $labels.instance }})
+            description: |-
+              All Traefik services are down
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+#       - alert: TraefikHighHttp4xxErrorRateService
+#         expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+#         for: 1m
+#         labels:
+#           severity: critical
+#         annotations:
+#           summary: Traefik high HTTP 4xx error rate service (instance {{ $labels.instance }})
+#           description: |-
+#             Traefik service 4xx error rate is above 5%
+#               VALUE = {{ $value }}
+#               LABELS = {{ $labels }}
+        - alert: TraefikHighHttp5xxErrorRateService
+          expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: Traefik high HTTP 5xx error rate service (instance {{ $labels.instance }})
+            description: |-
+              Traefik service 5xx error rate is above 5%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+#
+# legacy leverege alert
+#
+        - alert: TraefikConfigReloadFailed
+          expr: traefik_config_last_reload_failure > 0
+          for: 1m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Traefik config failed to reload"
+            description: "New config file for Traefik is invalid or Traefik encountered an error"

package/src/helmup.sh

@@ -696,7 +696,7 @@ kind: VolumeSnapshotClass
 metadata:
   name: cnpg-snapshotclass
 driver: pd.csi.storage.gke.io
-deletionPolicy: Delete
+deletionPolicy: Retain
 EOSNAPSC
 
   sleep 2 # hold up processing for a moment to allow IAM mods to propagate

package/src/helm-charts/prom-operator/rules/awesome-k8s.yaml

@@ -1,326 +0,0 @@
-# https://samber.github.io/awesome-prometheus-alerts/rules#kubernetes
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: awesome-k8s-rules
-  namespace: prometheus
-
-spec:
-  groups:
-  - name: Awesome Kubernetes
-    rules:
-    - alert: KubernetesNodeNotReady
-      expr: kube_node_status_condition{condition="Ready",status="true"} == 0
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node not ready (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has been unready for a long time\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeMemoryPressure
-      expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node memory pressure (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has MemoryPressure condition\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeDiskPressure
-      expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node disk pressure (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has DiskPressure condition\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeNetworkUnavailable
-      expr: kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node network unavailable (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has NetworkUnavailable condition\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeOutOfPodCapacity
-      expr: sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid, instance) group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) (kube_node_status_allocatable{resource="pods"}) * 100 > 90
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Node out of pod capacity (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} is out of pod capacity\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesContainerOomKiller
-#     expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
-#     for: 0m
-#     labels:
-#       severity: warning
-#     annotations:
-#       summary: Kubernetes Container oom killer (instance {{ $labels.instance }})
-#       description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesJobFailed
-      expr: kube_job_status_failed > 0
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Job failed (instance {{ $labels.instance }})
-        description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesJobNotStarting
-      expr: kube_job_status_active == 0 and kube_job_status_failed == 0 and kube_job_status_succeeded == 0 and (time() - kube_job_status_start_time) > 600
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Job not starting (instance {{ $labels.instance }})
-        description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} did not start for 10 minutes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesCronjobSuspended
-      expr: kube_cronjob_spec_suspend != 0
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes CronJob suspended (instance {{ $labels.instance }})
-        description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPersistentvolumeclaimPending
-      expr: kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes PersistentVolumeClaim pending (instance {{ $labels.instance }})
-        description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesVolumeOutOfDiskSpace
-      expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }})
-        description: "Volume is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesVolumeFullInFourDays
-#     expr: predict_linear(kubelet_volume_stats_available_bytes[6h:5m], 4 * 24 * 3600) < 0
-#     for: 0m
-#     labels:
-#       severity: critical
-#     annotations:
-#       summary: Kubernetes Volume full in four days (instance {{ $labels.instance }})
-#       description: "Volume under {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPersistentvolumeError
-      expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0
-      for: 0m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }})
-        description: "Persistent volume {{ $labels.persistentvolume }} is in bad state\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetDown
-      expr: kube_statefulset_replicas != kube_statefulset_status_replicas_ready > 0
-      for: 1m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes StatefulSet down (instance {{ $labels.instance }})
-        description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} went down\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesHpaScaleInability
-#     expr: (kube_horizontalpodautoscaler_spec_max_replicas - kube_horizontalpodautoscaler_status_desired_replicas) * on (horizontalpodautoscaler,namespace) (kube_horizontalpodautoscaler_status_condition{condition="ScalingLimited", status="true"} == 1) == 0
-#     for: 2m
-#     labels:
-#       severity: warning
-#     annotations:
-#       summary: Kubernetes HPA scale inability (instance {{ $labels.instance }})
-#       description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to scale\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesHpaMetricsUnavailability
-      expr: kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive"} == 1
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes HPA metrics unavailability (instance {{ $labels.instance }})
-        description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to collect metrics\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesHpaScaleMaximum
-#     expr: (kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas) and (kube_horizontalpodautoscaler_spec_max_replicas > 1) and (kube_horizontalpodautoscaler_spec_min_replicas != kube_horizontalpodautoscaler_spec_max_replicas)
-#     for: 2m
-#     labels:
-#       severity: info
-#     annotations:
-#       summary: Kubernetes HPA scale maximum (instance {{ $labels.instance }})
-#       description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit maximum number of desired pods\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesHpaUnderutilized
-      expr: max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3
-      for: 0m
-      labels:
-        severity: info
-      annotations:
-        summary: Kubernetes HPA underutilized (instance {{ $labels.instance }})
-        description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is constantly at minimum replicas for 50% of the time. Potential cost saving here.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPodNotHealthy
-      expr: sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"}) > 0
-      for: 15m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Pod not healthy (instance {{ $labels.instance }})
-        description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-running state for longer than 15 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPodCrashLooping
-      expr: increase(kube_pod_container_status_restarts_total[1m]) > 3
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes pod crash looping (instance {{ $labels.instance }})
-        description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesReplicasetReplicasMismatch
-      expr: kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes ReplicaSet replicas mismatch (instance {{ $labels.instance }})
-        description: "ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDeploymentReplicasMismatch
-      expr: kube_deployment_spec_replicas != kube_deployment_status_replicas_available
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Deployment replicas mismatch (instance {{ $labels.instance }})
-        description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replicas mismatch\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetReplicasMismatch
-      expr: kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})
-        description: "StatefulSet does not match the expected number of replicas.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDeploymentGenerationMismatch
-      expr: kube_deployment_status_observed_generation != kube_deployment_metadata_generation
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Deployment generation mismatch (instance {{ $labels.instance }})
-        description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has failed but has not been rolled back.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetGenerationMismatch
-      expr: kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes StatefulSet generation mismatch (instance {{ $labels.instance }})
-        description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has failed but has not been rolled back.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetUpdateNotRolledOut
-      expr: max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes StatefulSet update not rolled out (instance {{ $labels.instance }})
-        description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDaemonsetRolloutStuck
-      expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes DaemonSet rollout stuck (instance {{ $labels.instance }})
-        description: "Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled or not ready\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDaemonsetMisscheduled
-      expr: kube_daemonset_status_number_misscheduled > 0
-      for: 1m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes DaemonSet misscheduled (instance {{ $labels.instance }})
-        description: "Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    # Threshold should be customized for each cronjob name.
-    - alert: KubernetesCronjobTooLong
-      expr: time() - kube_cronjob_next_schedule_time > 3600
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes CronJob too long (instance {{ $labels.instance }})
-        description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesJobSlowCompletion
-      expr: kube_job_spec_completions - kube_job_status_succeeded - kube_job_status_failed > 0
-      for: 12h
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Job slow completion (instance {{ $labels.instance }})
-        description: "Kubernetes Job {{ $labels.namespace }}/{{ $labels.job_name }} did not complete in time.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesApiServerErrors
-      expr: sum(rate(apiserver_request_total{job="apiserver",code=~"(?:5..)"}[1m])) by (instance, job) / sum(rate(apiserver_request_total{job="apiserver"}[1m])) by (instance, job) * 100 > 3
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes API server errors (instance {{ $labels.instance }})
-        description: "Kubernetes API server is experiencing high error rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesApiClientErrors
-      expr: (sum(rate(rest_client_requests_total{code=~"(4|5).."}[1m])) by (instance, job) / sum(rate(rest_client_requests_total[1m])) by (instance, job)) * 100 > 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes API client errors (instance {{ $labels.instance }})
-        description: "Kubernetes API client is experiencing high error rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesClientCertificateExpiresNextWeek
-      expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes client certificate expires next week (instance {{ $labels.instance }})
-        description: "A client certificate used to authenticate to the apiserver is expiring next week.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesClientCertificateExpiresSoon
-      expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60
-      for: 0m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes client certificate expires soon (instance {{ $labels.instance }})
-        description: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesApiServerLatency
-      expr: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~"(?:CONNECT|WATCHLIST|WATCH|PROXY)"} [10m])) WITHOUT (subresource)) > 1
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes API server latency (instance {{ $labels.instance }})
-        description: "Kubernetes API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

package/src/helm-charts/prom-operator/rules/kubernetes-rules.yaml

@@ -1,109 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: kubernetes-rules
-  namespace: prometheus
-
-spec:
-  groups:
-  - name: Kubernetes
-    rules:
-    - alert: KubernetesNodeNotReady
-      expr: kube_node_status_condition{condition="Ready",status="true"} == 0
-      for: 5m
-      labels:
-        severity: page
-        priority: P3
-      annotations:
-        summary: "Kubernetes Node is not ready"
-        description: "Node {{ $labels.node }} is stuck in unready state for more than 5m"
-        dashboard: "Check GKE or EKS node status"
-
-    - alert: KubernetesMemoryPressure
-      expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
-      for: 5m
-      labels:
-        severity: error
-      annotations:
-        summary: "Kubernetes memory pressure"
-        description: "{{ $labels.node }} has a MemoryPressure condition"
-        dashboard: "Check GKE or EKS node status"
-
-    - alert: KubernetesDiskPressure
-      expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
-      for: 5m
-      labels:
-        severity: error
-      annotations:
-        summary: "Kubernetes disk pressure"
-        description: "{{ $labels.node }} has DiskPressure condition"
-        dashboard: "Check GKE or EKS node status"
-
-    - alert: KubernetesOutOfDisk
-      expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1
-      for: 5m
-      labels:
-        severity: error
-      annotations:
-        summary: "Kubernetes out of disk"
-        description: "{{ $labels.node }} has is out of disk space"
-        dashboard: "Check GKE or EKS node status"
-
-    - alert: KubernetesVolumeOutOfDiskSpace
-      expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
-      for: 5m
-      labels:
-        severity: error
-        priority: P2
-      annotations:
-        summary: "Kubernetes PVC Free Space < 10%"
-        description: "PVC volume {{ $labels.persistentvolumeclaim }} is almost full - down to {{ $value | humanize }}% available"
-
-#   - alert: KubernetesVolumeFullInFourDays
-#     expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0
-#     for: 5m
-#     labels:
-#       severity: error
-#     annotations:
-#       summary: "Kubernetes Volume full in four days "
-#       description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available."
-
-    - alert: KubernetesPersistentvolumeError
-      expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
-      for: 5m
-      labels:
-        severity: error
-      annotations:
-        summary: "Kubernetes PersistentVolume error "
-        description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is stuck pending"
-        dashboard: "Check GKE or EKS PVC/storage status"
-
-    - alert: KubernetesHpaScaleCapability
-      expr: kube_hpa_status_desired_replicas >= kube_hpa_spec_max_replicas
-      for: 15m
-      labels:
-        severity: warning
-      annotations:
-        summary: "Kubernetes HPA scale capability"
-        description: "Kubernetes has scaled {{ $labels.hpa }} to max number of scaled pods."
-
-    - alert: KubernetesPodNotHealthy
-      expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed"} == 1
-      for: 20m
-      labels:
-        severity: error
-      annotations:
-        summary: "Kubernetes Pod not healthy"
-        description: "Pod {{ $labels.pod }} has been in a non-ready state for longer than 20 minutes."
-        dashboard: "Check the pod status using the Kubernetes UI"
-
-    - alert: KubernetesPodCrashLooping
-      expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5
-      for: 5m
-      labels:
-        severity: page
-        priority: P2
-      annotations:
-        summary: "Kubernetes pod crash looping"
-        description: "{{ $labels.pod }} is in a crash loop"
-        dashboard: "Check the pod status using the Kubernetes UI"