@leverege/build-tools 2.21.13 → 2.22.0

package/src/system-charts/prometheus-local.yaml

@@ -0,0 +1,786 @@
+alertmanager:
+  affinity:
+    nodeAffinity: 
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - preference:
+          matchExpressions:
+          - key: cloud.google.com/gke-nodepool
+            operator: In
+            values:
+            - database
+        weight: 100
+  tolerations:
+  - key: "database"
+    operator: "Equal"
+    value: "true"
+    effect: "NoSchedule"
+
+  strategy:
+    type: Recreate
+
+alertmanagerFiles:
+  alertmanager.yml:
+    receivers:
+      - name: slack
+        slack_configs:
+          - channel: "OVH:<SLACK_CHANNEL>"
+            api_url: "OVH:<SLACK_HOOK_URL>"
+            username: "OVH:<PROJECT_NAME>"
+            fallback: "OVH:<PROJECT_NAME> - {{ .CommonAnnotations.summary }}"
+            title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"
+            title_link: "https://bitbucket.org/leverege/sre-playbook/src/master/"
+            send_resolved: true
+            text: |-
+              {{ range .Alerts }}
+                *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
+                *Description:* {{ .Annotations.description }}
+                *Dashboard:* {{ .Annotations.dashboard }}
+              {{ end }}
+      - name: opsgenie
+        opsgenie_configs:
+          - api_key: "OVH:<OPSGENIE_APIKEY>"
+            priority: "{{ .CommonLabels.priority }}"
+            tags: "{{ range .Alerts }}{{ .Labels.severity }},{{ end }}" 
+            description: |-
+              {{ range .Alerts }}
+                *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
+                *Description:* {{ .Annotations.description }}
+                *Dashboard:* {{ .Annotations.dashboard }}
+              {{ end }}
+
+    route:
+      group_wait: 10s
+      group_interval: 5m
+      receiver: slack
+      routes:
+      - match:
+          severity: page
+        receiver: opsgenie
+      repeat_interval: 6h
+
+serverFiles:
+  alerting_rules.yml:
+    groups:
+      # UNCOMMENT IF YOU NEED ANOMALY DETECTION
+      # - name: PubSubRules
+      #   rules:
+      #     - record: job:pubsub_message_size:rate5m
+      #       expr: sum by (topic_id) (rate(stackdriver_pubsub_topic_pubsub_googleapis_com_topic_message_sizes_count[5m]) )
+      #     - record: job:pubsub_message_size:rate5m:avg_over_time_1w
+      #       expr: avg_over_time(job:pubsub_message_size:rate5m[1w])
+      #     - record: job:pubsub_message_size:rate5m:stddev_over_time_1w
+      #       expr: stddev_over_time(job:pubsub_message_size:rate5m[1w])
+
+      - name: Kubernetes
+        rules:
+          - alert: KubernetesNodeNotReady
+            expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+            for: 5m
+            labels:
+              severity: page
+              priority: P3
+            annotations:
+              summary: "Kubernetes Node is not ready"
+              description: "Node {{ $labels.node }} is stuck in unready state for more than 5m"
+              dashboard: "Check GKE or EKS node status"
+
+          - alert: KubernetesMemoryPressure
+            expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Kubernetes memory pressure"
+              description: "{{ $labels.node }} has a MemoryPressure condition"
+              dashboard: "Check GKE or EKS node status"
+
+          - alert: KubernetesDiskPressure
+            expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Kubernetes disk pressure"
+              description: "{{ $labels.node }} has DiskPressure condition"
+              dashboard: "Check GKE or EKS node status"
+
+          - alert: KubernetesOutOfDisk
+            expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Kubernetes out of disk"
+              description: "{{ $labels.node }} has is out of disk space"
+              dashboard: "Check GKE or EKS node status"
+
+          - alert: KubernetesVolumeOutOfDiskSpace
+            expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 20
+            for: 5m
+            labels:
+              severity: page
+              priority: P2
+            annotations:
+              summary: "Kubernetes Volume out of disk space"
+              description: "Volume is almost full (< 10% left)"
+              dashboard: "OVH:<MONITORING>/pvc"
+
+#         - alert: KubernetesVolumeFullInFourDays
+#           expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0
+#           for: 5m
+#           labels:
+#             severity: error
+#           annotations:
+#             summary: "Kubernetes Volume full in four days "
+#             description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available."
+#             dashboard: "OVH:<MONITORING>/pvc"
+
+          - alert: KubernetesPersistentvolumeError
+            expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Kubernetes PersistentVolume error "
+              description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is stuck pending"
+              dashboard: "Check GKE or EKS PVC/storage status"
+
+          - alert: KubernetesHpaScaleCapability
+            expr: kube_hpa_status_desired_replicas >= kube_hpa_spec_max_replicas
+            for: 15m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Kubernetes HPA scale capability"
+              description: "Kubernetes has scaled {{ $labels.hpa }} to max number of scaled pods."
+              dashboard: "OVH:<MONITORING>/k8-deployment"
+
+          - alert: KubernetesPodNotHealthy
+            expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed"} == 1
+            for: 10m
+            labels:
+              severity: error
+            annotations:
+              summary: "Kubernetes Pod not healthy"
+              description: "Pod {{ $labels.pod }} has been in a non-ready state for longer than a 10m."
+              dashboard: "Check the pod status using the Kubernetes UI"
+
+          - alert: KubernetesPodCrashLooping
+            expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5
+            for: 5m
+            labels:
+              severity: page
+              priority: P2
+            annotations:
+              summary: "Kubernetes pod crash looping"
+              description: "{{ $labels.pod }} is in a crash loop"
+              dashboard: "Check the pod status using the Kubernetes UI"
+
+      - name: Prometheus
+        rules: 
+          - alert: targets_down
+            expr: up{target_env!="preemptibles"} == 0
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: Prometheus target missing
+              description: "Prometheus failed to scrape {{ $labels.app }}"
+              dashboard: "See prometheus.md in SRE Playbook"
+
+          - alert: PrometheusConfigurationReloadFailure
+            expr: prometheus_config_last_reload_successful != 1
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Prometheus server configuration reload failure"
+              description: "Prometheus server failed to reload server configs (e.g. alerts, rules)"
+              dashboard: "See prometheus.md in SRE Playbook"
+
+          - alert: PrometheusAlertmanagerConfigurationReloadFailure
+            expr: alertmanager_config_last_reload_successful != 1
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Prometheus AlertManager configuration reload failure"
+              description: "AlertManager failed to reload alert configs (e.g. slack, opsgenie)"
+              dashboard: "See prometheus.md in SRE Playbook"
+
+          - alert: PrometheusTooManyRestarts
+            expr: changes(process_start_time_seconds{job=~"prometheus|alertmanager"}[15m]) > 2
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Prometheus is in a crash loop"
+              description: "Prometheus {{ $labels.component }} has restarted more than twice in the last 15 minutes. It might be crashlooping."
+              dashboard: "See prometheus.md in SRE Playbook"
+
+          - alert: PrometheusNotConnectedToAlertmanager
+            expr: prometheus_notifications_alertmanagers_discovered < 1
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Prometheus not connected to alertmanager"
+              description: "Prometheus cannot connect the alertmanager - new alerts may not be detected"
+              dashboard: "See prometheus.md in SRE Playbook"
+
+          - alert: PrometheusNotificationsBacklog
+            expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Prometheus notifications backlog"
+              description: "The Prometheus notification queue has not been empty for 10 minutes"
+              dashboard: "See prometheus.md in SRE Playbook"
+
+          - alert: PrometheusAlertmanagerNotificationFailing
+            expr: rate(alertmanager_notifications_failed_total[1m]) > 0
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Prometheus AlertManager notification failing"
+              description: "Alertmanager is failing to send notifications"
+              dashboard: "See prometheus.md in SRE Playbook"
+
+      - name: GCP
+        rules:
+          - alert: StackdriverFailedScrape
+            expr: absent(stackdriver_pubsub_topic_pubsub_googleapis_com_topic_message_sizes_count) == 1
+            for: 10m 
+            labels:
+              severity: warning
+            annotations:
+              summary: "Stackdriver metrics unavailable"
+              description: "Prometheus failed to scrape Stackdriver metrics"
+              dashboard: "Use Stackdriver Monitoring on GCP UI"
+          - alert: HighFirebaseLoad
+            expr: stackdriver_firebase_namespace_firebasedatabase_googleapis_com_io_database_load > 0.9
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "High Firebase load detected"
+              description: "Firebase load is above 90% (current value is: {{ $value | humanize }}%)"
+              dashboard: "OVH:<MONITORING>/firebase"   
+          - alert: PubSubUndeliveredMessages
+            expr: sum(stackdriver_pubsub_subscription_pubsub_googleapis_com_subscription_num_undelivered_messages) by (subscription_id) > 1000
+            for: 5m 
+            labels:
+              severity: page
+              priority: P3
+            annotations:
+              summary: "High Number of Undelivered Messages on PubSub"
+              description: "Undelivered message count on topic {{$labels.subscription_id}} is greater than 1000"
+              dashboard: "OVH:<MONITORING>/stackdriver"
+          - alert: PubSubOldUnackedMessages
+            expr: stackdriver_pubsub_subscription_pubsub_googleapis_com_subscription_oldest_unacked_message_age > 900
+            for: 5m 
+            labels:
+              severity: warning
+            annotations:
+              summary: "Old messages left unacked on PubSub"
+              description: "Messages on {{$labels.subscription_id}} has been unacked for more than 15m"
+              dashboard: "OVH:<MONITORING>/stackdriver"
+
+      - name: Apps
+        rules:
+          - alert: frequent_app_restarts
+            expr: rate(nodejs_app_exit_count[15m]) * 60 * 5 > 5
+            for: 1m
+            labels:
+              severity: page
+              priority: P2
+            annotations:
+              summary: Apps are restarting frequently
+              description: "{{ $labels.app }} is in a crash loop"
+              dashboard: "OVH:<MONITORING>/stackdriver"
+
+      - name: MySQL
+        rules:
+          - alert: MysqlDown
+            expr: mysql_up == 0
+            for: 5m
+            labels:
+              severity: page
+              priority: P3
+            annotations:
+              summary: "MySQL down (instance {{ $labels.instance }})"
+              description: "MySQL pod is down. Look into manually restarting."
+              dashboard: "OVH:<MONITORING>/mysql"
+
+          - alert: MysqlSlowQueries
+            expr: mysql_global_status_slow_queries > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "MySQL slow queries detected"
+              description: "MySQL server is detected {{ $value }} slow queries."
+              dashboard: "OVH:<MONITORING>/mysql"
+
+          - alert: MysqlRestarted
+            expr: mysql_global_status_uptime < 60
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "MySQL restarted"
+              description: "MySQL has just been restarted, less than one minute ago."
+              dashboard: "OVH:<MONITORING>/mysql"
+
+      - name: Postgres
+        rules:
+          - alert: PostgresqlDown
+            expr: pg_up == 0
+            for: 5m
+            labels:
+              severity: page
+              priority: P3
+            annotations:
+              summary: "Postgresql down"
+              description: "Postgresql instance is down: check {{ $labels.release }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlRestarted
+            expr: time() - pg_postmaster_start_time_seconds < 60
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Postgresql restarted"
+              description: "Postgresql restarted: check {{ $labels.release }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlExporterError
+            expr: pg_exporter_last_scrape_error > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql exporter error for {{ $labels.release }}"
+              description: "Postgresql exporter is showing errors. Metrics may be outdated"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlReplicationLag
+            expr: (pg_replication_lag > 10) and ON(instance) (pg_replication_is_replica == 1)
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql replication lag"
+              description: "PostgreSQL replication lag is going up (> 10s): check {{ $labels.release }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlTableNotVaccumed
+            expr: time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql table not vaccumed"
+              description: "Table has not been vaccum for 24 hours: check {{ $labels.release }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlTableNotAnalyzed
+            expr: time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql table not analyzed (instance {{ $labels.instance }})"
+              description: "Table has not been analyzed for 24 hours: check {{ $labels.release }} - {{ $labels.datname }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlDeadLocks
+            expr: rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql dead locks"
+              description: "PostgreSQL has dead-locks: check {{ $labels.release }} - {{ $labels.datname }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlSlowQueries
+            expr: pg_slow_queries > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql slow queries"
+              description: "PostgreSQL executing slow queries"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlHighRollbackRate
+            expr: rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql high rollback rate on {{ $labels.release }} - {{ $labels.datname }}"
+              description: "Ratio of transactions being aborted compared to committed is > 2 %"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlCommitRateLow
+            expr: rate(pg_stat_database_xact_commit[1m]) < 10
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Postgresql commit rate low on {{ $labels.release }} - {{ $labels.datname }}"
+              description: "Postgres seems to be processing very few transactions"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlLowXidConsumption
+            expr: rate(pg_txid_current[1m]) < 5
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql low XID consumption on {{ $labels.release }} - {{ $labels.datname }}"
+              description: "Postgresql seems to be consuming transaction IDs very slowly"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqllowXlogConsumption
+            expr: rate(pg_xlog_position_bytes[1m]) < 100
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresqllow XLOG consumption on {{ $labels.release }} - {{ $labels.datname }}"
+              description: "Postgres seems to be consuming XLOG very slowly"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlWaleReplicationStopped
+            expr: rate(pg_xlog_position_bytes[1m]) == 0
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Postgresql WAL-E replication stopped on {{ $labels.release }} - {{ $labels.datname }}"
+              description: "WAL-E replication seems to be stopped"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlHighRateStatementTimeout
+            expr: rate(postgresql_errors_total{type="statement_timeout"}[5m]) > 3
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Postgresql high rate statement timeout"
+              description: "Postgres transactions showing high rate of statement timeouts on {{ $labels.release }} - {{ $labels.datname }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlHighRateDeadlock
+            expr: rate(postgresql_errors_total{type="deadlock_detected"}[1m]) * 60 > 1
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Postgresql high rate deadlock"
+              description: "Postgres detected deadlocks on {{ $labels.release }} - {{ $labels.datname }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlReplicationLabBytes
+            expr: (pg_xlog_position_bytes and pg_replication_is_replica == 0) - on (environment) group_right(instance) (pg_xlog_position_bytes and pg_replication_is_replica == 1) > 1e+09
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Postgresql replication lab bytes"
+              description: "Postgres Replication lag (in bytes) is high on {{ $labels.release }} - {{ $labels.datname }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlTooManyDeadTuples
+            expr: ((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1)
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Postgresql too many dead tuples"
+              description: "PostgreSQL dead tuples is too large ( {{ $value }} dead tuples) on {{ $labels.release }} - {{ $labels.datname }}"
+              dashboard: "OVH:<MONITORING>/postgres"
+
+          - alert: PostgresqlTooManyLocksAcquired
+            expr: ((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Postgresql too many locks acquired on {{ $labels.statefulset.kubernetes.io_pod_name }}"
+              description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction."
+              dashboard: "OVH:<MONITORING>/postgres"
+
+      - name: Redis
+        rules:             
+          - alert: RedisDown
+            expr: redis_up == 0
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Redis down (instance {{ $labels.instance }})"
+              description: "Redis instance is down"
+              dashboard: "OVH:<MONITORING>/redis"
+
+          - alert: RedisMissingMaster
+            expr: count(redis_instance_info{role="master"}) == 0
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Redis missing master (instance {{ $labels.instance }})"
+              description: "Redis cluster has no node marked as master."
+              dashboard: "OVH:<MONITORING>/redis"
+
+          - alert: RedisTooManyMasters
+            expr: count(redis_instance_info{role="master"}) > 1
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Redis too many masters (instance {{ $labels.instance }})"
+              description: "Redis cluster has too many nodes marked as master."
+              dashboard: "OVH:<MONITORING>/redis"
+
+          - alert: RedisDisconnectedSlaves
+            expr: count without (instance, job) (redis_connected_slaves) - sum without (instance, job) (redis_connected_slaves) - 1 > 1
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Redis disconnected slaves (instance {{ $labels.instance }})"
+              description: "Redis not replicating for all slaves. Consider reviewing the redis replication status."
+              dashboard: "OVH:<MONITORING>/redis"
+
+          - alert: RedisReplicationBroken
+            expr: delta(redis_connected_slaves[1m]) < 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Redis replication broken (instance {{ $labels.instance }})"
+              description: "Redis instance lost a slave"
+              dashboard: "OVH:<MONITORING>/redis"
+
+          - alert: RedisClusterFlapping
+            expr: changes(redis_connected_slaves[5m]) > 2
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Redis cluster flapping (instance {{ $labels.instance }})"
+              description: "Changes have been detected in Redis replica connection. This can occur when replica nodes lose connection to the master and reconnect (a.k.a flapping)."
+              dashboard: "OVH:<MONITORING>/redis"
+
+          - alert: RedisOutOfMemory
+            expr: redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Redis out of memory (instance {{ $labels.instance }})"
+              description: "Redis is running out of memory ( {{ $value }}% )"
+              dashboard: "OVH:<MONITORING>/redis"
+
+          - alert: RedisRejectedConnections
+            expr: increase(redis_rejected_connections_total[1m]) > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Redis rejected connections (instance {{ $labels.instance }})"
+              description: "Some connections to Redis has been rejected. Please check {{ $labels.app }}"
+              dashboard: "OVH:<MONITORING>/redis"
+      
+      - name: Traefik Alerts
+        rules:
+          - alert: TraefikConfigReloadFailed
+            expr: traefik_config_last_reload_failure > 0
+            for: 1m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Traefik config failed to reload"
+              description: "New config file for Traefik is invalid or Traefik encountered an error"
+
+          - alert: TraefikHighHttp4xxError
+            expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
+              description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
+              dashboard: "OVH:<MONITORING>/traefik"
+
+          - alert: TraefikHighHttp5xxError
+            expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[5m])) by (service) / sum(rate(traefik_service_requests_total[5m])) by (service) * 100 > 5
+            for: 5m
+            labels:
+              severity: page
+              priority: P3
+            annotations:
+              summary: "Traefik high HTTP 5xx error rate on {{ $labels.service }}"
+              description: "Traefik backend 5xx error rate is above 5% (currently at {{ $value }}%)"
+              dashboard: "OVH:<MONITORING>/traefik"
+
+      - name: Elasticsearch Alerts
+        rules:
+          - alert: ElasticsearchHeapUsageTooHigh
+            expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90
+            for: 5m
+            labels:
+              severity: page
+              priority: P3
+            annotations:
+              summary: "Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})"
+              description: "The heap usage is over 90% for 5m (currently at {{ $value }}% )"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchHeapUsageWarning
+            expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Elasticsearch Heap Usage warning (instance {{ $labels.instance }})"
+              description: "The heap usage is over 80% for 5m (currently at {{ $value }}% )"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchDiskSpaceLow
+            expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Elasticsearch disk space low (instance {{ $labels.instance }})"
+              description: "The disk usage is over 80% (currently at {{ $value }}% )"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchDiskOutOfSpace
+            expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10
+            for: 5m
+            labels:
+              severity: page
+              priority: P3
+            annotations:
+              summary: "Elasticsearch disk out of space (instance {{ $labels.instance }})"
+              description: "The disk usage is over 90% (currently at {{ $value }}% )"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchClusterRed
+            expr: elasticsearch_cluster_health_status{color="red"} == 1
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Elasticsearch Cluster Red (instance {{ $labels.instance }})"
+              description: "Elastic Cluster is in an unhealthy state"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchClusterYellow
+            expr: elasticsearch_cluster_health_status{color="yellow"} == 1
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Elasticsearch Cluster Yellow (instance {{ $labels.instance }})"
+              description: "Elastic Cluster is in an unhealthy state"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchRelocationShards
+            expr: elasticsearch_cluster_health_relocating_shards > 0
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Elasticsearch relocation shards (instance {{ $labels.instance }})"
+              description: "Number of relocation shards for 20 min is {{ $value }}"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchInitializingShards
+            expr: elasticsearch_cluster_health_initializing_shards > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Elasticsearch initializing shards (instance {{ $labels.instance }})"
+              description: "Number of initializing shards for 10 min is {{ $value }}"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchUnassignedShards
+            expr: elasticsearch_cluster_health_unassigned_shards > 0
+            for: 5m
+            labels:
+              severity: error
+            annotations:
+              summary: "Elasticsearch unassigned shards (instance {{ $labels.instance }})"
+              description: "Number of unassigned shards for 2 min is {{ $value }}"
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+
+          - alert: ElasticsearchPendingTasks
+            expr: elasticsearch_cluster_health_number_of_pending_tasks > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Elasticsearch pending tasks (instance {{ $labels.instance }})"
+              description: "Number of pending tasks for 10 min is {{ $value }}. Cluster is working slowly."
+              dashboard: "OVH:<MONITORING>/elasticsearch"
+      - name: Velero Alerts
+        rules:
+          - alert: VeleroBackupFailure
+            expr: (time() - velero_backup_last_successful_timestamp{schedule!=""} > 108000) and (velero_backup_success_total{schedule!=""}  != 0)
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Velero schedule {{ $labels.schedule }} backup failed to execute properly"
+              description: "It has been over 30 hours since the last successful backup"
+              dashboard: "OVH:<MONITORING>/velero"
+          - alert: NoVeleroBackups
+            expr: velero_backup_success_total{schedule!=""}  == 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Velero has not made any backups yet"
+              description: "Velero has been broken since cluster initialization because there is no successful backups around"
+              dashboard: "OVH:<MONITORING>/velero"
+
+server:
+  affinity:
+    nodeAffinity: 
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - preference:
+          matchExpressions:
+          - key: cloud.google.com/gke-nodepool
+            operator: In
+            values:
+            - database
+        weight: 100
+  tolerations:
+  - key: "database"
+    operator: "Equal"
+    value: "true"
+    effect: "NoSchedule"  
+  retention: "30d"
+  persistentVolume:
+    size: 50Gi
+  resources:
+    limits:
+      memory: 4Gi
+    requests:
+      memory: 2Gi
+  strategy:
+    type: Recreate
+
+pushgateway:
+  enabled: false