@leverege/build-tools 2.54.0 → 2.55.0

package/src/helm-charts/prom-operator/rules/traefik-rules.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: traefik-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Traefik Alerts
+    rules:
+    - alert: TraefikConfigReloadFailed
+      expr: traefik_config_last_reload_failure > 0
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Traefik config failed to reload"
+        description: "New config file for Traefik is invalid or Traefik encountered an error"
+
+#   - alert: TraefikHighHttp4xxError
+#     expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
+#     for: 5m
+#     labels:
+#       severity: error
+#     annotations:
+#       summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
+#       description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
+#       dashboard: "OVH:<MONITORING>/traefik"

package/src/helm-charts/prom-operator/rules/velero-rules.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: velero-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Velero
+    rules:
+    - alert: VeleroBackupFailure
+      expr: (time() - velero_backup_last_successful_timestamp{schedule_version=~"2.[0]"} > 108000) and (velero_backup_success_total{schedule!=""}  != 0)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Velero schedule {{ $labels.schedule }} backup failed to execute properly"
+        description: "It has been over 30 hours since the last successful backup"
+        dashboard: "OVH:<MONITORING>/velero"
+    - alert: NoVeleroBackups
+      expr: velero_backup_success_total{schedule!=""}  == 0
+      for: 30h
+      labels:
+        severity: warning
+      annotations:
+        summary: "Velero has not made any backups yet"
+        description: "No backups have successfully been made since velero was initialized (or restarted), go check it out ASAP"
+        dashboard: "OVH:<MONITORING>/velero"

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -0,0 +1,52 @@
+#
+# to force a reinstall of this service simply remove or rename this
+# stackdriver-exporter subdirectory and rerun helmup stackdriver-exporter
+#
+fullnameOverride: "prometheus-stack-stackdriver-metrics"
+
+stackdriver:
+  projectId: "OVH:<PROJECT_NAME>"
+  metrics:
+    typePrefixes: "pubsub.googleapis.com/subscription/oldest_unacked_message_age,pubsub.googleapis.com/subscription/num_undelivered_messages,pubsub.googleapis.com/subscription/pull_ack_request_count,pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,pubsub.googleapis.com/topic/message_sizes,pubsub.googleapis.com/topic/send_request_count,firebasedatabase.googleapis.com/io/database_load,firebasedatabase.googleapis.com/network/sent_bytes_count,firebasedatabase.googleapis.com/network/active_connections"
+
+# https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter?modal=values&path=serviceMonitor
+serviceMonitor:
+  enabled: true
+  namespace: prometheus
+  interval: 20s
+
+annotations:
+  prometheus.io/port: "9255"
+  prometheus.io/scrape: "true" # this is a string annotation - so quoted!
+
+tolerations:
+- key: "preemptible"
+  operator: "Equal"
+  value: "true"
+  effect: "NoSchedule"
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: target-env
+          operator: In
+          values:
+          - preemptible
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 32Mi
+
+# NOTE to future selves - the exporter's helm chart does not have a clue
+# regarding workload identity, which means the gcloud and k8s service
+# accounts have to be managed outside of the chart by helmup. In order to get
+# the linkages from the SA to the service we specify the SA name here, but
+# disable the creation of the SA since it will have already have been created
+# and bound by the helmup installStackdriverExporterEnvironment function.
+#
+serviceAccount:
+  create : false # this is a flag to helm - so unquoted ... da F?!
+  name: "stackdriver-exporter"

package/src/helm-charts/redis/redis-local.yaml

@@ -70,3 +70,6 @@ networkPolicy:
 
 metrics:
   enabled: true
+  serviceMonitor:
+    enabled: false # set to true for prometheus operator
+    namespace: "prometheus"

package/src/helmup.sh

@@ -296,7 +296,7 @@ function installStackdriverExporterEnvironment() {
   # Add necessary helm repositories - note that stable is deprecated
   addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-  local SDEXP_NS="monitoring"
+  local SDEXP_NS=${1:-monitoring}
   local SDEXP_SA="stackdriver-exporter"
   local SDEXP_EM="$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
 
@@ -306,7 +306,8 @@ function installStackdriverExporterEnvironment() {
   printf "\n*** Removing the previous $SDEXP_SA installation...\n"
   helm uninstall -n $SDEXP_NS $SDEXP_SA                 &> $DEVNULL
   kubectl delete serviceaccounts -n $SDEXP_NS $SDEXP_SA &> $DEVNULL
-  gcloud --quiet iam service-accounts delete $SDEXP_EM  &> $DEVNULL
+  printf "!!! Skipping deletion of the gcloud SA $SDEXP_EM\n"
+# gcloud --quiet iam service-accounts delete $SDEXP_EM  &> $DEVNULL
 
   # NOTE: The k8s SA != GCP SA - the former is created by the helm chart
   # that installs the stackdriver-exporter, the latter is handled here in
@@ -321,7 +322,7 @@ function installStackdriverExporterEnvironment() {
 
   printf "\n*** Binding IAM role of $SDEXP_NS viewer to $SDEXP_SA in $GCP_PROJECT_ID\n"
   gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \
-    --role "roles/$SDEXP_NS.viewer" \
+    --role "roles/monitoring.viewer" \
     --member "serviceAccount:$SDEXP_EM" &> $DEVNULL
   sleep 2 # give the IAM binding a chance to complete before moving on
 
@@ -1002,7 +1003,13 @@ CATBACKUP
         bootstrapLocalSetup $SERVICE
         ;;
 
-      "prometheus"|"prometheus-operator")
+      "prometheus")
+        installStackdriverExporterEnvironment
+        bootstrapLocalSetup $SERVICE
+        ;;
+
+      "prom-operator")
+        installStackdriverExporterEnvironment prometheus
         bootstrapLocalSetup $SERVICE
         ;;
 

package/src/overwhelm.mjs

@@ -206,6 +206,13 @@ const doReplacements = ( yamls, replacements ) => {
     process.exit( 1 )
   }
 
+  // Eat any doubly double quoted values such that
+  //   ""<ovh replace>"" -> "<ovh replace>"
+  // but
+  //   someDefault: ""
+  // remains unchanged
+  replYaml = replYaml.replace( /""([^"]+)""/g, '"$1"' )
+
   return replYaml
 }
 
@@ -307,7 +314,7 @@ ahoy.forEach( ( dir ) => {
   }
 } )
 
-// we can also apply replaceables to any top level .ovh files to support top level config maps
+// we can also apply replaceables to all .ovh files in the project
 glob.sync( '**/*.ovh' ).forEach( ( cfgmap ) => {
   const target = cfgmap.replace( /.ovh$/, '' )
   let valuesOut = fs.readFileSync( cfgmap, { encoding : 'utf-8' } )