@leverege/build-tools 2.65.1 → 2.66.1

package/src/helm-charts/prom-operator/rules/traefik-rules.yaml

@@ -1,27 +1,54 @@
+# https://samber.github.io/awesome-prometheus-alerts/rules#traefik
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: traefik-rules
+  name: embedded-exporter-v2
   namespace: prometheus
-
 spec:
   groups:
-  - name: Traefik Alerts
-    rules:
-    - alert: TraefikConfigReloadFailed
-      expr: traefik_config_last_reload_failure > 0
-      for: 1m
-      labels:
-        severity: warning
-      annotations:
-        summary: "Traefik config failed to reload"
-        description: "New config file for Traefik is invalid or Traefik encountered an error"
-
-#   - alert: TraefikHighHttp4xxError
-#     expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
-#     for: 5m
-#     labels:
-#       severity: error
-#     annotations:
-#       summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
-#       description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
+    - name: EmbeddedExporterV2-rules
+      rules:
+        - alert: TraefikServiceDown
+          expr: count(traefik_service_server_up) by (service) == 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Traefik service down (instance {{ $labels.instance }})
+            description: |-
+              All Traefik services are down
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+#       - alert: TraefikHighHttp4xxErrorRateService
+#         expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+#         for: 1m
+#         labels:
+#           severity: critical
+#         annotations:
+#           summary: Traefik high HTTP 4xx error rate service (instance {{ $labels.instance }})
+#           description: |-
+#             Traefik service 4xx error rate is above 5%
+#               VALUE = {{ $value }}
+#               LABELS = {{ $labels }}
+        - alert: TraefikHighHttp5xxErrorRateService
+          expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: Traefik high HTTP 5xx error rate service (instance {{ $labels.instance }})
+            description: |-
+              Traefik service 5xx error rate is above 5%
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+#
+# legacy leverege alert
+#
+        - alert: TraefikConfigReloadFailed
+          expr: traefik_config_last_reload_failure > 0
+          for: 1m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Traefik config failed to reload"
+            description: "New config file for Traefik is invalid or Traefik encountered an error"

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -7,7 +7,25 @@ fullnameOverride: "prometheus-stack-stackdriver-metrics"
 stackdriver:
   projectId: "${PROJECT_NAME}"
   metrics:
-    typePrefixes: "pubsub.googleapis.com/subscription/oldest_unacked_message_age,pubsub.googleapis.com/subscription/num_undelivered_messages,pubsub.googleapis.com/subscription/pull_ack_request_count,pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,pubsub.googleapis.com/topic/message_sizes,pubsub.googleapis.com/topic/send_request_count,firebasedatabase.googleapis.com/io/database_load,firebasedatabase.googleapis.com/network/sent_bytes_count,firebasedatabase.googleapis.com/network/active_connections"
+    typePrefixes:  >
+      pubsub.googleapis.com/subscription/oldest_unacked_message_age,
+      pubsub.googleapis.com/subscription/num_undelivered_messages,
+      pubsub.googleapis.com/subscription/pull_ack_request_count,
+      pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,
+      pubsub.googleapis.com/topic/message_sizes,
+      pubsub.googleapis.com/topic/send_request_count,
+      firebasedatabase.googleapis.com/io/database_load,
+      firebasedatabase.googleapis.com/network/sent_bytes_count,
+      firebasedatabase.googleapis.com/network/active_connections,
+      container.googleapis.com/accelerator/duty_cycle,
+      container.googleapis.com/accelerator/memory_usage,
+      container.googleapis.com/accelerator/request_count,
+      agent.googleapis.com/memory/bytes_used,
+      agent.googleapis.com/cpu/utilization,
+      storage.googleapis.com/network/received_bytes_count,
+      storage.googleapis.com/network/sent_bytes_count,
+      monitoring.googleapis.com/uptime_check/check_passed,
+      quota.googleapis.com/allocation/usage
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter?modal=values&path=serviceMonitor
 serviceMonitor:
@@ -15,6 +33,8 @@ serviceMonitor:
   namespace: prometheus
   interval: "30s"
   relabelings:
+    # Legacy labelmap used to preserve compatibility with pre-operator Grafana dashboards
+    # Can be removed once legacy Prometheus stack is deprecated
     - action: labelmap
       regex: __meta_kubernetes_pod_label_(.+)
     - action: labelmap

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.11.3"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.11.5"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/values.yaml \

package/src/helm-charts/traefik/helmup.plugin

@@ -17,7 +17,7 @@ TRAEFIK_NAMESPACE="traefik"
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="34"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="35"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/values.yaml \

package/src/helm-charts/velero/helmup.plugin

@@ -25,7 +25,7 @@ velero schedule delete --all --confirm &> /dev/null
 helm upgrade \
      --install velero vmware-tanzu/velero \
      --namespace velero --create-namespace \
-     --values velero/velero-local.yaml \
+     --values velero/values-local.yaml \
      --version $VELERO_CHART_VERSION $HELM_WHAT \
      --set configuration.backupStorageLocation[0].bucket="$GCE_BUCKET" \
      --set configuration.backupStorageLocation[0].config.serviceAccount="$GCE_SA_EMAIL" \

package/src/helm-charts/velero/{velero-local.yaml → values-local.yaml}

@@ -71,7 +71,7 @@ resources:
     cpu: 250m
     memory: 128Mi
   limits:
-    cpu: 300m
+    cpu: 500m
     memory: 256Mi
 
 # Annotations to add to the Velero deployment's. Optional.

package/src/helmup.sh

@@ -696,13 +696,17 @@ kind: VolumeSnapshotClass
 metadata:
   name: cnpg-snapshotclass
 driver: pd.csi.storage.gke.io
-deletionPolicy: Delete
+deletionPolicy: Retain
 EOSNAPSC
 
   sleep 2 # hold up processing for a moment to allow IAM mods to propagate
 }
 
-function installCronCnpgCleanupSnapshot() {
+# this function is a little convoluted in the way it attempts to install the
+# scripts and yaml necessary for the CNPG cleanup activities - the idea is to
+# be able to update the scripts without touching anything else since the target
+# directory is the cnpg-operator setup
+function installCnpgCleanupCronjob() {
   CNPG_OPERATOR_DIR="cnpg-operator"
   if [ ! -d "$CNPG_OPERATOR_DIR" ];
   then
@@ -713,23 +717,52 @@ CNPG_OPERATOR_MISSING
     exit 1
   fi
 
-  CNPG_CLEANUP_CRONJOB_YAML="cnpg-cleanup-snapshot-cronjob.yaml"
-  CNPG_CLEANUP_CRONJOB="$CNPG_OPERATOR_DIR/$CNPG_CLEANUP_CRONJOB_YAML"
-  if [ ! -f "$CNPG_CLEANUP_CRONJOB" ];
+  # create the destination bucket for k8s cronjob scripts
+  BUCKET_SUFFIX="k8s-cronjobs"
+  makeUniformBucketWithCors $BUCKET_SUFFIX
+
+  CNPG_CLEANUP_CRONJOB_OVH="cnpg-cleanup-cronjob.yaml.ovh"
+  CNPG_CLEANUP_CRONJOB="$CNPG_OPERATOR_DIR/$CNPG_CLEANUP_CRONJOB_OVH"
+  CNPG_CLEANUP_SCRIPT_BASH="cnpg-cleanup-script.sh"
+  CNPG_CLEANUP_SCRIPT="$CNPG_OPERATOR_DIR/$CNPG_CLEANUP_SCRIPT_BASH"
+  if [ ! -f "$CNPG_CLEANUP_SCRIPT" ];
   then
     cat<<CNPG_FETCH_CLEANUP_CRONJOB_YAML
-  `color y "Missing $CNPG_CLEANUP_CRONJOB_YAML - grabbing a copy from build-tools"`
+  `color y "Missing $CNPG_CLEANUP_SCRIPT_BASH - updating local cron setup scripts"`
 
 CNPG_FETCH_CLEANUP_CRONJOB_YAML
     CNPG_OP_ROOT="$(build-tools --reporoot)/src/helm-charts/cnpg-operator"
-    cp $CNPG_OP_ROOT/$CNPG_CLEANUP_CRONJOB_YAML $CNPG_CLEANUP_CRONJOB
+    cp $CNPG_OP_ROOT/$CNPG_CLEANUP_CRONJOB_OVH $CNPG_CLEANUP_CRONJOB
+    cp $CNPG_OP_ROOT/$CNPG_CLEANUP_SCRIPT_BASH $CNPG_CLEANUP_SCRIPT
   fi
 
+  # copy / update the script in the bucket referenced in the cronjob
+  printf "\ngsutil cp $CNPG_CLEANUP_SCRIPT gs://$GCP_PROJECT_ID-$BUCKET_SUFFIX\n"
+  gsutil cp $CNPG_CLEANUP_SCRIPT gs://$GCP_PROJECT_ID-$BUCKET_SUFFIX
+
+  local CNPG_NAMESPACE="cnpg-operands"
+  local CNPG_CLEANUP_SA="cnpg-cleanup-sa"
+
+  gcloud iam service-accounts create "$CNPG_CLEANUP_SA" \
+    --project="$GCP_PROJECT_ID" \
+    --description="Used for cronjob cleanup with Workload Identity" \
+    --display-name="CNPG Cleanup SA" &> $DEVNULL
+    warnOnError $? "the GCP SA $CNPG_CLEANUP_SA may already exist"
+
+  gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \
+    --project="$GCP_PROJECT_ID" \
+    --member="serviceAccount:$CNPG_CLEANUP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
+    --role="roles/storage.objectViewer"
+
+  bindWorkloadIdentity "$CNPG_CLEANUP_SA" "$CNPG_NAMESPACE"
+
+  CNPG_CLEANUP_CRONJOB_YAML="$CNPG_OPERATOR_DIR/$(basename $CNPG_CLEANUP_CRONJOB_OVH .ovh)"
+  overwhelm -g # allow ovh -> yaml to occur
   cat<<CNPG_CLEANUP_CRONJOB_APPLY
-  `color g "Creating cronjob =>"``color y " kubectl apply -f $CNPG_CLEANUP_CRONJOB"`
+  `color g "Creating cronjob =>"``color y " kubectl apply -f $CNPG_CLEANUP_CRONJOB_YAML"`
 
 CNPG_CLEANUP_CRONJOB_APPLY
-  kubectl apply -f $CNPG_CLEANUP_CRONJOB
+  kubectl apply -f $CNPG_CLEANUP_CRONJOB_YAML
 }
 
 function ensureCnpgSecret() {
@@ -1001,7 +1034,7 @@ CATBACKUP
         ;;
 
       "cnpg-cronjob")
-        installCronCnpgCleanupSnapshot
+        installCnpgCleanupCronjob
         ;;
 
       "cronZombieKiller"|"zombie-killer")

package/src/templates/cloudBuildSteps.hbs

@@ -1,4 +1,8 @@
 steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    entrypoint: 'bash'
+    args: ['-c', 'docker pull {{imageName}}:latest || exit 0']
+
   - name: 'gcr.io/cloud-builders/docker'
     args:
       - 'build' 
@@ -6,8 +10,8 @@ steps:
       - '-t'
       - '{{imageName}}:{{imageVersion}}'
       - '--cache-from'
-      - '{{imageName}}:{{imageVersion}}'
+      - '{{imageName}}:latest'
       - '.'
     
 images:
-  - '{{imageName}}:{{imageVersion}}'
+  - '{{imageName}}:{{imageVersion}}'

package/src/helm-charts/cnpg-operator/cnpg-cleanup-snapshot-cronjob.yaml

@@ -1,73 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cnpg-cleanup-sa
-  namespace: cnpg-operands
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: cnpg-cleanup-role
-  namespace: cnpg-operands
-rules:
-  - apiGroups: ["postgresql.cnpg.io"]
-    resources: ["backups"]
-    verbs: ["get", "list", "delete"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: cnpg-cleanup-rolebinding
-  namespace: cnpg-operands
-subjects:
-  - kind: ServiceAccount
-    name: cnpg-cleanup-sa
-    namespace: cnpg-operands
-roleRef:
-  kind: Role
-  name: cnpg-cleanup-role
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: cnpg-cleanup-snapshot
-  namespace: cnpg-operands
-spec:
-  schedule: "0 13 * * *" # 13 UTC is ~8AM EST
-  successfulJobsHistoryLimit: 2
-  jobTemplate:
-    spec:
-      template:
-        spec:
-          serviceAccountName: cnpg-cleanup-sa
-          restartPolicy: Never
-          containers:
-            - name: cnpg-cleanup-snapshots
-              image: bitnami/kubectl:latest
-              command:
-                - "/bin/bash"
-                - "-c"
-                - |
-                  echo "Starting CNPG Cleanup"
-                  CURRENT_DATE=$(date +%s)
-                  RETENTION_DAYS=21
-                  SNAPSHOTS_JSON=$(kubectl get backups.postgresql.cnpg.io -n cnpg-operands -o json)
-                  echo "$SNAPSHOTS_JSON" | jq -c '.items[]' | while read -r snapshot
-                  do
-                    NAME=$(echo "$snapshot" | jq -r '.metadata.name')
-                    CREATION_TIMESTAMP=$(echo "$snapshot" | jq -r '.metadata.creationTimestamp')
-                    if [[ "$NAME" != *"-backup-"* ]];
-                    then
-                      echo "Skipping manual snapshot: $NAME"
-                      continue
-                    fi
-                    SNAPSHOT_DATE=$(date -d "$CREATION_TIMESTAMP" +%s)
-                    AGE=$(( (CURRENT_DATE - SNAPSHOT_DATE) / 86400 ))
-                    if [ "$AGE" -gt "$RETENTION_DAYS" ];
-                    then
-                      echo "Deleting snapshot: $NAME (Age: $AGE days)"
-                      kubectl delete backups.postgresql.cnpg.io "$NAME" -n cnpg-operands 2> /dev/null
-                    fi
-                  done
-                  echo "Cleanup complete."

package/src/helm-charts/prom-operator/rules/awesome-k8s.yaml

@@ -1,326 +0,0 @@
-# https://samber.github.io/awesome-prometheus-alerts/rules#kubernetes
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: awesome-k8s-rules
-  namespace: prometheus
-
-spec:
-  groups:
-  - name: Awesome Kubernetes
-    rules:
-    - alert: KubernetesNodeNotReady
-      expr: kube_node_status_condition{condition="Ready",status="true"} == 0
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node not ready (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has been unready for a long time\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeMemoryPressure
-      expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node memory pressure (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has MemoryPressure condition\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeDiskPressure
-      expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node disk pressure (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has DiskPressure condition\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeNetworkUnavailable
-      expr: kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Node network unavailable (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} has NetworkUnavailable condition\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesNodeOutOfPodCapacity
-      expr: sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid, instance) group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) (kube_node_status_allocatable{resource="pods"}) * 100 > 90
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Node out of pod capacity (instance {{ $labels.instance }})
-        description: "Node {{ $labels.node }} is out of pod capacity\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesContainerOomKiller
-#     expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
-#     for: 0m
-#     labels:
-#       severity: warning
-#     annotations:
-#       summary: Kubernetes Container oom killer (instance {{ $labels.instance }})
-#       description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesJobFailed
-      expr: kube_job_status_failed > 0
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Job failed (instance {{ $labels.instance }})
-        description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesJobNotStarting
-      expr: kube_job_status_active == 0 and kube_job_status_failed == 0 and kube_job_status_succeeded == 0 and (time() - kube_job_status_start_time) > 600
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Job not starting (instance {{ $labels.instance }})
-        description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} did not start for 10 minutes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesCronjobSuspended
-      expr: kube_cronjob_spec_suspend != 0
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes CronJob suspended (instance {{ $labels.instance }})
-        description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPersistentvolumeclaimPending
-      expr: kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes PersistentVolumeClaim pending (instance {{ $labels.instance }})
-        description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesVolumeOutOfDiskSpace
-      expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }})
-        description: "Volume is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesVolumeFullInFourDays
-#     expr: predict_linear(kubelet_volume_stats_available_bytes[6h:5m], 4 * 24 * 3600) < 0
-#     for: 0m
-#     labels:
-#       severity: critical
-#     annotations:
-#       summary: Kubernetes Volume full in four days (instance {{ $labels.instance }})
-#       description: "Volume under {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPersistentvolumeError
-      expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0
-      for: 0m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }})
-        description: "Persistent volume {{ $labels.persistentvolume }} is in bad state\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetDown
-      expr: kube_statefulset_replicas != kube_statefulset_status_replicas_ready > 0
-      for: 1m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes StatefulSet down (instance {{ $labels.instance }})
-        description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} went down\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesHpaScaleInability
-#     expr: (kube_horizontalpodautoscaler_spec_max_replicas - kube_horizontalpodautoscaler_status_desired_replicas) * on (horizontalpodautoscaler,namespace) (kube_horizontalpodautoscaler_status_condition{condition="ScalingLimited", status="true"} == 1) == 0
-#     for: 2m
-#     labels:
-#       severity: warning
-#     annotations:
-#       summary: Kubernetes HPA scale inability (instance {{ $labels.instance }})
-#       description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to scale\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesHpaMetricsUnavailability
-      expr: kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive"} == 1
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes HPA metrics unavailability (instance {{ $labels.instance }})
-        description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to collect metrics\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-#   - alert: KubernetesHpaScaleMaximum
-#     expr: (kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas) and (kube_horizontalpodautoscaler_spec_max_replicas > 1) and (kube_horizontalpodautoscaler_spec_min_replicas != kube_horizontalpodautoscaler_spec_max_replicas)
-#     for: 2m
-#     labels:
-#       severity: info
-#     annotations:
-#       summary: Kubernetes HPA scale maximum (instance {{ $labels.instance }})
-#       description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit maximum number of desired pods\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesHpaUnderutilized
-      expr: max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3
-      for: 0m
-      labels:
-        severity: info
-      annotations:
-        summary: Kubernetes HPA underutilized (instance {{ $labels.instance }})
-        description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is constantly at minimum replicas for 50% of the time. Potential cost saving here.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPodNotHealthy
-      expr: sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"}) > 0
-      for: 15m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Pod not healthy (instance {{ $labels.instance }})
-        description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-running state for longer than 15 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesPodCrashLooping
-      expr: increase(kube_pod_container_status_restarts_total[1m]) > 3
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes pod crash looping (instance {{ $labels.instance }})
-        description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesReplicasetReplicasMismatch
-      expr: kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes ReplicaSet replicas mismatch (instance {{ $labels.instance }})
-        description: "ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDeploymentReplicasMismatch
-      expr: kube_deployment_spec_replicas != kube_deployment_status_replicas_available
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes Deployment replicas mismatch (instance {{ $labels.instance }})
-        description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replicas mismatch\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetReplicasMismatch
-      expr: kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})
-        description: "StatefulSet does not match the expected number of replicas.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDeploymentGenerationMismatch
-      expr: kube_deployment_status_observed_generation != kube_deployment_metadata_generation
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Deployment generation mismatch (instance {{ $labels.instance }})
-        description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has failed but has not been rolled back.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetGenerationMismatch
-      expr: kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation
-      for: 10m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes StatefulSet generation mismatch (instance {{ $labels.instance }})
-        description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has failed but has not been rolled back.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesStatefulsetUpdateNotRolledOut
-      expr: max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes StatefulSet update not rolled out (instance {{ $labels.instance }})
-        description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDaemonsetRolloutStuck
-      expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0
-      for: 10m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes DaemonSet rollout stuck (instance {{ $labels.instance }})
-        description: "Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled or not ready\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesDaemonsetMisscheduled
-      expr: kube_daemonset_status_number_misscheduled > 0
-      for: 1m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes DaemonSet misscheduled (instance {{ $labels.instance }})
-        description: "Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    # Threshold should be customized for each cronjob name.
-    - alert: KubernetesCronjobTooLong
-      expr: time() - kube_cronjob_next_schedule_time > 3600
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes CronJob too long (instance {{ $labels.instance }})
-        description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesJobSlowCompletion
-      expr: kube_job_spec_completions - kube_job_status_succeeded - kube_job_status_failed > 0
-      for: 12h
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes Job slow completion (instance {{ $labels.instance }})
-        description: "Kubernetes Job {{ $labels.namespace }}/{{ $labels.job_name }} did not complete in time.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesApiServerErrors
-      expr: sum(rate(apiserver_request_total{job="apiserver",code=~"(?:5..)"}[1m])) by (instance, job) / sum(rate(apiserver_request_total{job="apiserver"}[1m])) by (instance, job) * 100 > 3
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes API server errors (instance {{ $labels.instance }})
-        description: "Kubernetes API server is experiencing high error rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesApiClientErrors
-      expr: (sum(rate(rest_client_requests_total{code=~"(4|5).."}[1m])) by (instance, job) / sum(rate(rest_client_requests_total[1m])) by (instance, job)) * 100 > 1
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes API client errors (instance {{ $labels.instance }})
-        description: "Kubernetes API client is experiencing high error rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesClientCertificateExpiresNextWeek
-      expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes client certificate expires next week (instance {{ $labels.instance }})
-        description: "A client certificate used to authenticate to the apiserver is expiring next week.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesClientCertificateExpiresSoon
-      expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60
-      for: 0m
-      labels:
-        severity: critical
-      annotations:
-        summary: Kubernetes client certificate expires soon (instance {{ $labels.instance }})
-        description: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: KubernetesApiServerLatency
-      expr: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~"(?:CONNECT|WATCHLIST|WATCH|PROXY)"} [10m])) WITHOUT (subresource)) > 1
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Kubernetes API server latency (instance {{ $labels.instance }})
-        description: "Kubernetes API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"