@leverege/build-tools 2.54.1 → 2.55.0

package/src/helm-charts/prom-operator/rules/kubernetes-rules.yaml.off

@@ -0,0 +1,112 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: kubernetes-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Kubernetes
+    rules:
+    - alert: KubernetesNodeNotReady
+      expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+      for: 5m
+      labels:
+        severity: page
+        priority: P3
+      annotations:
+        summary: "Kubernetes Node is not ready"
+        description: "Node {{ $labels.node }} is stuck in unready state for more than 5m"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesMemoryPressure
+      expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes memory pressure"
+        description: "{{ $labels.node }} has a MemoryPressure condition"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesDiskPressure
+      expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes disk pressure"
+        description: "{{ $labels.node }} has DiskPressure condition"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesOutOfDisk
+      expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes out of disk"
+        description: "{{ $labels.node }} has is out of disk space"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesVolumeOutOfDiskSpace
+      expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
+      for: 5m
+      labels:
+        severity: error
+        priority: P2
+      annotations:
+        summary: "Kubernetes PVC Free Space < 10%"
+        description: "PVC volume {{ $labels.persistentvolumeclaim }} is almost full - down to {{ $value | humanize }}% available"
+        dashboard: "OVH:<MONITORING>/k8s-pvc/pvc-usage?var-volume={{ $labels.persistentvolumeclaim }}"
+
+#   - alert: KubernetesVolumeFullInFourDays
+#     expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0
+#     for: 5m
+#     labels:
+#       severity: error
+#     annotations:
+#       summary: "Kubernetes Volume full in four days "
+#       description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available."
+#       dashboard: "OVH:<MONITORING>/pvc"
+
+    - alert: KubernetesPersistentvolumeError
+      expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes PersistentVolume error "
+        description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is stuck pending"
+        dashboard: "Check GKE or EKS PVC/storage status"
+
+    - alert: KubernetesHpaScaleCapability
+      expr: kube_hpa_status_desired_replicas >= kube_hpa_spec_max_replicas
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Kubernetes HPA scale capability"
+        description: "Kubernetes has scaled {{ $labels.hpa }} to max number of scaled pods."
+        dashboard: "OVH:<MONITORING>/k8-deployment"
+
+    - alert: KubernetesPodNotHealthy
+      expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed"} == 1
+      for: 20m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes Pod not healthy"
+        description: "Pod {{ $labels.pod }} has been in a non-ready state for longer than 20 minutes."
+        dashboard: "Check the pod status using the Kubernetes UI"
+
+    - alert: KubernetesPodCrashLooping
+      expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5
+      for: 5m
+      labels:
+        severity: page
+        priority: P2
+      annotations:
+        summary: "Kubernetes pod crash looping"
+        description: "{{ $labels.pod }} is in a crash loop"
+        dashboard: "Check the pod status using the Kubernetes UI"

package/src/helm-charts/prom-operator/rules/leverege-rules.yaml

@@ -0,0 +1,20 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: leverege-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Leverege
+    rules:
+    - alert: frequent_app_restarts
+      expr: rate(nodejs_app_exit_count[15m]) * 60 * 5 > 5
+      for: 1m
+      labels:
+        severity: page
+        priority: P2
+      annotations:
+        summary: Apps are restarting frequently
+        description: "{{ $labels.app }} is in a crash loop"
+        dashboard: "OVH:<MONITORING>/stackdriver"

package/src/helm-charts/prom-operator/rules/postgres-rules.yaml

@@ -0,0 +1,190 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: postgres-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Postgres
+    rules:
+    - alert: PostgresqlDown
+      expr: pg_up == 0
+      for: 5m
+      labels:
+        severity: page
+        priority: P3
+      annotations:
+        summary: "Postgresql down"
+        description: "Postgresql instance is down: check {{ $labels.release }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlRestarted
+      expr: time() - pg_postmaster_start_time_seconds < 60
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql restarted"
+        description: "Postgresql restarted: check {{ $labels.release }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlExporterError
+      expr: pg_exporter_last_scrape_error > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql exporter error for {{ $labels.release }}"
+        description: "Postgresql exporter is showing errors. Metrics may be outdated"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlReplicationLag
+      expr: (pg_replication_lag > 10) and ON(instance) (pg_replication_is_replica == 1)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql replication lag"
+        description: "PostgreSQL replication lag is going up (> 10s): check {{ $labels.release }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTableNotVacuumed
+      expr: (pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql table not vacuumed"
+        description: "Table has not been vacuum for 24 hours: check {{ $labels.instance }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTableNotAutoAnalyzed
+      expr: (pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql table not analyzed (instance {{ $labels.instance }})"
+        description: "Table {{ $labels.relname }} has not been analyzed for 24 hours: check {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlDeadLocks
+      expr: rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql dead locks"
+        description: "PostgreSQL has dead-locks: check {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlSlowQueries
+      expr: pg_slow_queries > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql slow queries"
+        description: "PostgreSQL executing slow queries"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlHighRollbackRate
+      expr: rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql high rollback rate on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Ratio of transactions being aborted compared to committed is > 2 %"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlCommitRateLow
+      expr: rate(pg_stat_database_xact_commit[1m]) < 10
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql commit rate low on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Postgres seems to be processing very few transactions"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlLowXidConsumption
+      expr: rate(pg_txid_current[1m]) < 5
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql low XID consumption on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Postgresql seems to be consuming transaction IDs very slowly"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqllowXlogConsumption
+      expr: rate(pg_xlog_position_bytes[1m]) < 100
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresqllow XLOG consumption on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Postgres seems to be consuming XLOG very slowly"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlWaleReplicationStopped
+      expr: rate(pg_xlog_position_bytes[1m]) == 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql WAL-E replication stopped on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "WAL-E replication seems to be stopped"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlHighRateStatementTimeout
+      expr: rate(postgresql_errors_total{type="statement_timeout"}[5m]) > 3
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql high rate statement timeout"
+        description: "Postgres transactions showing high rate of statement timeouts on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlHighRateDeadlock
+      expr: rate(postgresql_errors_total{type="deadlock_detected"}[1m]) * 60 > 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql high rate deadlock"
+        description: "Postgres detected deadlocks on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlReplicationLabBytes
+      expr: (pg_xlog_position_bytes and pg_replication_is_replica == 0) - on (environment) group_right(instance) (pg_xlog_position_bytes and pg_replication_is_replica == 1) > 1e+09
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql replication lab bytes"
+        description: "Postgres Replication lag (in bytes) is high on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTooManyDeadTuples
+      expr: ((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql too many dead tuples"
+        description: "PostgreSQL dead tuples is too large ( {{ $value }} dead tuples) on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTooManyLocksAcquired
+      expr: ((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql too many locks acquired on {{ $labels.statefulset.kubernetes.io_pod_name }}"
+        description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction."
+        dashboard: "OVH:<MONITORING>/postgres"

package/src/helm-charts/prom-operator/rules/prometheus-rules.yaml

@@ -0,0 +1,79 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Prometheus
+    rules: 
+    - alert: targets_down
+      expr: up{target_env!="preemptibles"} == 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus target missing
+        description: "Prometheus failed to scrape {{ $labels.app }}"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusConfigurationReloadFailure
+      expr: prometheus_config_last_reload_successful != 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus server configuration reload failure"
+        description: "Prometheus server failed to reload server configs (e.g. alerts, rules)"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusAlertmanagerConfigurationReloadFailure
+      expr: alertmanager_config_last_reload_successful != 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus AlertManager configuration reload failure"
+        description: "AlertManager failed to reload alert configs (e.g. slack, opsgenie)"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusTooManyRestarts
+      expr: changes(process_start_time_seconds{job=~"prometheus|alertmanager"}[15m]) > 2
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus is in a crash loop"
+        description: "Prometheus {{ $labels.component }} has restarted more than twice in the last 15 minutes. It might be crashlooping."
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusNotConnectedToAlertmanager
+      expr: prometheus_notifications_alertmanagers_discovered < 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Prometheus not connected to alertmanager"
+        description: "Prometheus cannot connect the alertmanager - new alerts may not be detected"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusNotificationsBacklog
+      expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus notifications backlog"
+        description: "The Prometheus notification queue has not been empty for 10 minutes"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusAlertmanagerNotificationFailing
+      expr: rate(alertmanager_notifications_failed_total[1m]) > 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Prometheus AlertManager notification failing"
+        description: "Alertmanager is failing to send notifications"
+        dashboard: "See prometheus.md in SRE Playbook"

package/src/helm-charts/prom-operator/rules/prometheus-rules.yaml.off

@@ -0,0 +1,79 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Prometheus
+    rules: 
+    - alert: targets_down
+      expr: up{target_env!="preemptibles"} == 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus target missing
+        description: "Prometheus failed to scrape {{ $labels.app }}"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusConfigurationReloadFailure
+      expr: prometheus_config_last_reload_successful != 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus server configuration reload failure"
+        description: "Prometheus server failed to reload server configs (e.g. alerts, rules)"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusAlertmanagerConfigurationReloadFailure
+      expr: alertmanager_config_last_reload_successful != 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus AlertManager configuration reload failure"
+        description: "AlertManager failed to reload alert configs (e.g. slack, opsgenie)"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusTooManyRestarts
+      expr: changes(process_start_time_seconds{job=~"prometheus|alertmanager"}[15m]) > 2
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus is in a crash loop"
+        description: "Prometheus {{ $labels.component }} has restarted more than twice in the last 15 minutes. It might be crashlooping."
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusNotConnectedToAlertmanager
+      expr: prometheus_notifications_alertmanagers_discovered < 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Prometheus not connected to alertmanager"
+        description: "Prometheus cannot connect the alertmanager - new alerts may not be detected"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusNotificationsBacklog
+      expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus notifications backlog"
+        description: "The Prometheus notification queue has not been empty for 10 minutes"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusAlertmanagerNotificationFailing
+      expr: rate(alertmanager_notifications_failed_total[1m]) > 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Prometheus AlertManager notification failing"
+        description: "Alertmanager is failing to send notifications"
+        dashboard: "See prometheus.md in SRE Playbook"

package/src/helm-charts/prom-operator/rules/redis-rules.yaml

@@ -0,0 +1,90 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: redis-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Redis
+    rules:             
+    - alert: RedisDown
+      expr: redis_up == 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis down (instance {{ $labels.instance }})"
+        description: "Redis instance is down"
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisMissingMaster
+      expr: count(redis_instance_info{role="master"}) == 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis missing master (instance {{ $labels.instance }})"
+        description: "Redis cluster has no node marked as master."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisTooManyMasters
+      expr: count(redis_instance_info{role="master"}) > 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis too many masters (instance {{ $labels.instance }})"
+        description: "Redis cluster has too many nodes marked as master."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisDisconnectedSlaves
+      expr: count without (instance, job) (redis_connected_slaves) - sum without (instance, job) (redis_connected_slaves) - 1 > 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis disconnected slaves (instance {{ $labels.instance }})"
+        description: "Redis not replicating for all slaves. Consider reviewing the redis replication status."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisReplicationBroken
+      expr: delta(redis_connected_slaves[1m]) < 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis replication broken (instance {{ $labels.instance }})"
+        description: "Redis instance lost a slave"
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisClusterFlapping
+      expr: changes(redis_connected_slaves[5m]) > 2
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis cluster flapping (instance {{ $labels.instance }})"
+        description: "Changes have been detected in Redis replica connection. This can occur when replica nodes lose connection to the master and reconnect (a.k.a flapping)."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisOutOfMemory
+      expr: redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis out of memory (instance {{ $labels.instance }})"
+        description: "Redis is running out of memory ( {{ $value }}% )"
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisRejectedConnections
+      expr: increase(redis_rejected_connections_total[1m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis rejected connections (instance {{ $labels.instance }})"
+        description: "Some connections to Redis has been rejected. Please check {{ $labels.app }}"
+        dashboard: "OVH:<MONITORING>/redis"
+

package/src/helm-charts/prom-operator/rules/traefik-rules.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: traefik-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Traefik Alerts
+    rules:
+    - alert: TraefikConfigReloadFailed
+      expr: traefik_config_last_reload_failure > 0
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Traefik config failed to reload"
+        description: "New config file for Traefik is invalid or Traefik encountered an error"
+
+#   - alert: TraefikHighHttp4xxError
+#     expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
+#     for: 5m
+#     labels:
+#       severity: error
+#     annotations:
+#       summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
+#       description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
+#       dashboard: "OVH:<MONITORING>/traefik"

package/src/helm-charts/prom-operator/rules/velero-rules.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: velero-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Velero
+    rules:
+    - alert: VeleroBackupFailure
+      expr: (time() - velero_backup_last_successful_timestamp{schedule_version=~"2.[0]"} > 108000) and (velero_backup_success_total{schedule!=""}  != 0)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Velero schedule {{ $labels.schedule }} backup failed to execute properly"
+        description: "It has been over 30 hours since the last successful backup"
+        dashboard: "OVH:<MONITORING>/velero"
+    - alert: NoVeleroBackups
+      expr: velero_backup_success_total{schedule!=""}  == 0
+      for: 30h
+      labels:
+        severity: warning
+      annotations:
+        summary: "Velero has not made any backups yet"
+        description: "No backups have successfully been made since velero was initialized (or restarted), go check it out ASAP"
+        dashboard: "OVH:<MONITORING>/velero"

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -0,0 +1,52 @@
+#
+# to force a reinstall of this service simply remove or rename this
+# stackdriver-exporter subdirectory and rerun helmup stackdriver-exporter
+#
+fullnameOverride: "prometheus-stack-stackdriver-metrics"
+
+stackdriver:
+  projectId: "OVH:<PROJECT_NAME>"
+  metrics:
+    typePrefixes: "pubsub.googleapis.com/subscription/oldest_unacked_message_age,pubsub.googleapis.com/subscription/num_undelivered_messages,pubsub.googleapis.com/subscription/pull_ack_request_count,pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,pubsub.googleapis.com/topic/message_sizes,pubsub.googleapis.com/topic/send_request_count,firebasedatabase.googleapis.com/io/database_load,firebasedatabase.googleapis.com/network/sent_bytes_count,firebasedatabase.googleapis.com/network/active_connections"
+
+# https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter?modal=values&path=serviceMonitor
+serviceMonitor:
+  enabled: true
+  namespace: prometheus
+  interval: 20s
+
+annotations:
+  prometheus.io/port: "9255"
+  prometheus.io/scrape: "true" # this is a string annotation - so quoted!
+
+tolerations:
+- key: "preemptible"
+  operator: "Equal"
+  value: "true"
+  effect: "NoSchedule"
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: target-env
+          operator: In
+          values:
+          - preemptible
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 32Mi
+
+# NOTE to future selves - the exporter's helm chart does not have a clue
+# regarding workload identity, which means the gcloud and k8s service
+# accounts have to be managed outside of the chart by helmup. In order to get
+# the linkages from the SA to the service we specify the SA name here, but
+# disable the creation of the SA since it will have already have been created
+# and bound by the helmup installStackdriverExporterEnvironment function.
+#
+serviceAccount:
+  create : false # this is a flag to helm - so unquoted ... da F?!
+  name: "stackdriver-exporter"