@leverege/build-tools 2.54.1 → 2.55.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.54.1",
+  "version": "2.55.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -63,13 +63,13 @@
     "command-line-usage": "^7.0.3",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.4.0",
+    "execa": "^9.4.1",
     "glob": "^11.0.0",
     "handlebars": "^4.7.8",
-    "inquirer": "^12.0.0",
+    "inquirer": "^12.0.1",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^18.0.1",
+    "npm-registry-fetch": "^18.0.2",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
@@ -82,4 +82,4 @@
     "@leverege/eslint-config-leverege": "^4.2.0",
     "npm": "^10.9.0"
   }
-}
+}

package/registry-compass.yaml

@@ -0,0 +1,20 @@
+registry:
+  - root: us-docker.pkg.dev
+  - repositories:
+    - name: stack
+      charts:
+        - api-server
+        - authz-server
+        - emailer
+        - message-processor
+    - name: leverege
+      charts:
+        - pubsub-pulse
+        - pusher
+        - overdose
+    - name: cox-health
+      charts:
+        - actions-server
+        - analytics-server
+        - centrak-healthz
+        - centrak-ingestor

package/src/chart-compass.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/*
+ * chart-to-registry will...
+ */
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import { lt as semverLt } from 'semver'
+import YAML from 'js-yaml'
+
+import {
+  condir,
+  debug,
+  errorExit,
+  log,
+  warning,
+  getGitRootDirectory,
+  parsePackageJson,
+  parseHelmChart,
+  shellCmd } from './Utils.mjs'
+
+const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
+  /* eslint-disable max-len */
+  {
+    name : 'location',
+    type : String,
+    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
+  },
+  {
+    name : 'project',
+    type : String,
+    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
+  },
+  {
+    name : 'repository',
+    type : String,
+    description : '{green the target repository to receive the pushed chart}',
+  },
+  {
+    name : 'dry-run',
+    type : Boolean,
+    description : '{yellow perform everything except the actual chart push}',
+  },
+  {
+    name : 'help',
+    type : Boolean,
+    description : '{green display this help screen}',
+  },
+  /* eslint-enable max-len */
+]
+
+const sections = [
+  {
+    header : 'Leverege Helm Chart Compass (for helmup)',
+    content : `{green This tool helps helmup navigate the helm charts stored in the
+artifact-registries.}`
+  },
+  { header : 'Options',
+    optionList : commandLineOptions,
+  },
+]
+
+const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
+const usage = commandLineUsage( sections )
+
+if ( args.help ) {
+  log( usage )
+  process.exit( 0 )
+}
+
+/* eslint-disable no-underscore-dangle */
+if ( args._unknown ) {
+  log( usage )
+  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
+  process.exit( 1 )
+}
+/* eslint-enable no-underscore-dangle */
+
+const minNodejsVersion = '18.0.0'
+if ( semverLt( process.version, minNodejsVersion ) ) {
+  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
+}
+
+// First of all, fail if we are not in a git repository
+let gitRoot
+try {
+  gitRoot = await getGitRootDirectory()
+} catch ( error ) {
+  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
+}
+
+const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
+
+condir( { chartCompass }, '<==Navigation' )

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -15,24 +15,5 @@ helm upgrade --install elasticsearch8 $OCI_CHART \
              --values $LOCAL_ES_VALUES \
              --version $ELASTIC_CHART_VERSION $HELM_WHAT
 
-## Installing ES8 exporter
-addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
-helm upgrade --install elasticsearch8-exporter prometheus-community/prometheus-elasticsearch-exporter \
-             --namespace monitoring --create-namespace \
-             --version 6 \
-             -f - <<ELASTIC8_EXPORTER_CHART_MOD
-fullnameOverride: elasticsearch8-exporter
-
-service:
-  annotations:
-    prometheus.io/port: "9108"
-    prometheus.io/scrape: "true"
-es:
-  uri: http://elasticsearch8.elastic.svc.cluster.local:9200
-
-resources:
-  requests:
-    cpu: 50m
-    memory: 32Mi
-ELASTIC8_EXPORTER_CHART_MOD
-removeHelmRepo prometheus-community
+## Installing ES8 exporter into monitoring namespace
+installElasticsearchExporter

package/src/helm-charts/prom-operator/apply-rules

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Load the standard helper functions
+. `build-tools --bashfun`
+
+RULES_DIR="prom-operator/rules"
+
+[ ! -d "$RULES_DIR" ] && errorExit "Missing rules directory [$RULES_DIR]"
+
+for ruleset in $RULES_DIR/*.yaml; do
+  printf "  Applying rules => `color g $ruleset`\n"
+  kubectl apply -n prometheus -f $ruleset
+done

package/src/helm-charts/prom-operator/elasticsearch-exporter.yaml

@@ -0,0 +1,20 @@
+# https://artifacthub.io/packages/helm/prometheus-community/prometheus-elasticsearch-exporter?modal=values
+#
+fullnameOverride: prometheus-stack-elasticsearch8-metrics
+
+service:
+  annotations:
+    prometheus.io/port: "9108"
+    prometheus.io/scrape: "true"
+es:
+  uri: http://elasticsearch8.elastic.svc.cluster.local:9200
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 32Mi
+
+serviceMonitor:
+  enabled: true
+  namespace: prometheus
+  interval: 30s

package/src/helm-charts/prom-operator/gitignore

@@ -0,0 +1,2 @@
+prometheus-stack.yaml
+stackdriver-exporter.yaml

package/src/helm-charts/prom-operator/helmdn.plugin

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This will uninstall of the the operator components...
+helm uninstall -n prometheus prometheus-stack elasticsearch8-exporter stackdriver-exporter
+
+# including all CRDs and configs
+kubectl delete crd \
+  alertmanagerconfigs.monitoring.coreos.com \
+  alertmanagers.monitoring.coreos.com \
+  podmonitors.monitoring.coreos.com \
+  probes.monitoring.coreos.com \
+  prometheusagents.monitoring.coreos.com \
+  prometheuses.monitoring.coreos.com \
+  prometheusrules.monitoring.coreos.com \
+  scrapeconfigs.monitoring.coreos.com \
+  servicemonitors.monitoring.coreos.com \
+  thanosrulers.monitoring.coreos.com

package/src/helm-charts/prom-operator/helmup.plugin

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+showInstalling "The Prometheus Operator and Components"
+addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
+
+showInstalling "The Prometheus Operator (kube-prometheus-stack)"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="65"
+helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
+             --namespace prometheus --create-namespace \
+             --values  prom-operator/prometheus-stack.yaml \
+             --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT
+
+showInstalling "The Elasticsearch Exporter (prom-operator)"
+[ -z "$ELASTICSEARCH_EXPORTER_CHART_VERSION" ] && ELASTICSEARCH_EXPORTER_CHART_VERSION="4"
+helm upgrade --install elasticsearch8-exporter prometheus-community/prometheus-elasticsearch-exporter \
+             --namespace prometheus --create-namespace \
+             --values prom-operator/elasticsearch-exporter.yaml \
+             --version $ELASTICSEARCH_EXPORTER_CHART_VERSION $HELM_WHAT
+
+showInstalling "The Stackdriver Exporter (prom-operator)"
+[ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="4"
+helm upgrade --install stackdriver-exporter prometheus-community/prometheus-stackdriver-exporter \
+             --namespace prometheus --create-namespace \
+             --values prom-operator/stackdriver-exporter.yaml \
+             --version $STACKDRIVER_EXPORTER_CHART_VERSION $HELM_WHAT
+
+removeHelmRepo prometheus-community

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -0,0 +1,137 @@
+enabled: true
+kubeControllerManager:
+  enabled: false
+nodeExporter:
+  enabled: false
+defaultRules:
+  create: true
+  rules:
+    alertmanager: false
+    etcd: false
+    configReloaders: false
+    general: true
+    k8s: true
+    kubeApiserver: true  # GPT Keep an eye on API server availability
+    kubeApiserverAvailability: false
+    kubeApiserverSlos: false
+    kubelet: true  # GPT Consider monitoring kubelet status, especially on scaled nodes
+    kubeProxy: false
+    kubePrometheusGeneral: false
+    kubePrometheusNodeRecording: false
+    kubernetesApps: false
+    kubernetesResources: false
+    kubernetesStorage: false
+    kubernetesSystem: false
+    kubeScheduler: false
+    kubeStateMetrics: true
+    network: false
+    node: false
+    nodeExporterAlerting: false
+    nodeExporterRecording: false
+    prometheus: false
+    prometheusOperator: false
+
+prometheus:
+  prometheusSpec:
+    podMonitorSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+
+# default is using kube-dns which is usually sufficient for our needs but
+# can be switched over to CoreDNS if the need arises
+coreDns:
+  enabled: false
+
+# managed GKE does not provide direct access to the k8s scheduler so we
+# can safely disable this here - see the Stackdrive Monitoring for more
+kubeScheduler:
+  enabled: false
+
+grafana:
+  enabled: true
+  service:
+    port: 3000
+  # -- the grafana admin password
+  adminPassword: prom-operator
+  defaultDashboardsEnabled: false
+  sidecar:
+      dashboards:
+        enabled: true
+  extraObjects:
+    - apiVersion: traefik.io/v1alpha1
+      kind: IngressRoute
+      metadata:
+        name: grafana
+        namespace: prometheus
+      spec:
+        entryPoints:
+          - websecure
+        routes:
+          - match: Host(`OVH:<PROJECT_NAME>-monitoring.OVH:<HOST>.com`)
+            kind: Rule
+            services:
+            - name: prometheus-stack-grafana
+              port: 3000
+
+alertmanager:
+  enabled: true
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
+  tolerations:
+  - key: "database"
+    operator: "Equal"
+    value: "true"
+    effect: "NoSchedule"
+
+  strategy:
+    type: Recreate
+
+  config:
+    receivers:
+      - name: slack
+        slack_configs:
+          - channel: "OVH:<SLACK_CHANNEL>"
+            api_url: "OVH:<SLACK_HOOK_URL>"
+            username: "OVH:<PROJECT_NAME>"
+            fallback: "OVH:<PROJECT_NAME> - {{ .CommonAnnotations.summary }}"
+            title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"
+            title_link: "https://bitbucket.org/leverege/sre-playbook/src/master/"
+            send_resolved: true
+            text: |-
+              {{ range .Alerts }}
+                *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
+                *Description:* {{ .Annotations.description }}
+                *Dashboard:* {{ .Annotations.dashboard }}
+              {{ end }}
+      - name: opsgenie
+        opsgenie_configs:
+          - api_key: "OVH:<OPSGENIE_APIKEY>"
+            priority: "{{ .CommonLabels.priority }}"
+            tags: "{{ range .Alerts }}{{ .Labels.severity }},{{ end }}"
+            description: |-
+              {{ range .Alerts }}
+                *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
+                *Description:* {{ .Annotations.description }}
+                *Dashboard:* {{ .Annotations.dashboard }}
+              {{ end }}
+
+    route:
+      group_wait: 30s
+      group_interval: 5m
+      group_by:
+        - alertname
+        - cluster
+      receiver: slack
+      routes:
+      - match:
+          severity: page
+        receiver: opsgenie
+      repeat_interval: 6h

package/src/helm-charts/prom-operator/rules/elasticsearch-rules.yaml

@@ -0,0 +1,111 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: elasticsearch-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Elasticsearch Alerts
+    rules:
+    - alert: ElasticsearchHeapUsageTooHigh
+      expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90
+      for: 5m
+      labels:
+        severity: page
+        priority: P3
+      annotations:
+        summary: "Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})"
+        description: "The heap usage is over 90% for 5m (currently at {{ $value }}% )"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchHeapUsageWarning
+      expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Elasticsearch Heap Usage warning (instance {{ $labels.instance }})"
+        description: "The heap usage is over 80% for 5m (currently at {{ $value }}% )"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchDiskSpaceLow
+      expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Elasticsearch disk space low (instance {{ $labels.instance }})"
+        description: "The disk usage is over 80% (currently at {{ $value }}% )"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchDiskOutOfSpace
+      expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10
+      for: 5m
+      labels:
+        severity: page
+        priority: P3
+      annotations:
+        summary: "Elasticsearch disk out of space (instance {{ $labels.instance }})"
+        description: "The disk usage is over 90% (currently at {{ $value }}% )"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchClusterRed
+      expr: elasticsearch_cluster_health_status{color="red"} == 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Elasticsearch Cluster Red (instance {{ $labels.instance }})"
+        description: "Elastic Cluster is in an unhealthy state"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchClusterYellow
+      expr: elasticsearch_cluster_health_status{color="yellow"} == 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Elasticsearch Cluster Yellow (instance {{ $labels.instance }})"
+        description: "Elastic Cluster is in an unhealthy state"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchRelocationShards
+      expr: elasticsearch_cluster_health_relocating_shards > 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Elasticsearch relocation shards (instance {{ $labels.instance }})"
+        description: "Number of relocation shards for 20 min is {{ $value }}"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchInitializingShards
+      expr: elasticsearch_cluster_health_initializing_shards > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Elasticsearch initializing shards (instance {{ $labels.instance }})"
+        description: "Number of initializing shards for 10 min is {{ $value }}"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchUnassignedShards
+      expr: elasticsearch_cluster_health_unassigned_shards > 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Elasticsearch unassigned shards (instance {{ $labels.instance }})"
+        description: "Number of unassigned shards for 2 min is {{ $value }}"
+        dashboard: "OVH:<MONITORING>/elasticsearch"
+
+    - alert: ElasticsearchPendingTasks
+      expr: elasticsearch_cluster_health_number_of_pending_tasks > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Elasticsearch pending tasks (instance {{ $labels.instance }})"
+        description: "Number of pending tasks for 10 min is {{ $value }}. Cluster is working slowly."
+        dashboard: "OVH:<MONITORING>/elasticsearch"

package/src/helm-charts/prom-operator/rules/gcp-rules.yaml

@@ -0,0 +1,47 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: gcp-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: GCP
+    rules:
+    - alert: StackdriverFailedScrape
+      expr: absent(stackdriver_pubsub_topic_pubsub_googleapis_com_topic_message_sizes_count) == 1
+      for: 10m 
+      labels:
+        severity: warning
+      annotations:
+        summary: "Stackdriver metrics unavailable"
+        description: "Prometheus failed to scrape Stackdriver metrics"
+        dashboard: "Use Stackdriver Monitoring on GCP UI"
+    - alert: HighFirebaseLoad
+      expr: stackdriver_firebase_namespace_firebasedatabase_googleapis_com_io_database_load > 0.9
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "High Firebase load detected"
+        description: "Firebase load is above 90% (current value is: {{ $value | humanize }}%)"
+        dashboard: "OVH:<MONITORING>/firebase"   
+    - alert: PubSubUndeliveredMessages
+      expr: sum(stackdriver_pubsub_subscription_pubsub_googleapis_com_subscription_num_undelivered_messages) by (subscription_id) > 1000
+      for: 5m 
+      labels:
+        severity: page
+        priority: P3
+      annotations:
+        summary: "High Number of Undelivered Messages on PubSub"
+        description: "Undelivered message count on topic {{$labels.subscription_id}} is greater than 1000"
+        dashboard: "OVH:<MONITORING>/stackdriver"
+    - alert: PubSubOldUnackedMessages
+      expr: stackdriver_pubsub_subscription_pubsub_googleapis_com_subscription_oldest_unacked_message_age > 900
+      for: 5m 
+      labels:
+        severity: warning
+      annotations:
+        summary: "Old messages left unacked on PubSub"
+        description: "Messages on {{$labels.subscription_id}} has been unacked for more than 15m"
+        dashboard: "OVH:<MONITORING>/stackdriver"

package/src/helm-charts/prom-operator/rules/kubernetes-rules.yaml

@@ -0,0 +1,112 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: kubernetes-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Kubernetes
+    rules:
+    - alert: KubernetesNodeNotReady
+      expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+      for: 5m
+      labels:
+        severity: page
+        priority: P3
+      annotations:
+        summary: "Kubernetes Node is not ready"
+        description: "Node {{ $labels.node }} is stuck in unready state for more than 5m"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesMemoryPressure
+      expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes memory pressure"
+        description: "{{ $labels.node }} has a MemoryPressure condition"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesDiskPressure
+      expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes disk pressure"
+        description: "{{ $labels.node }} has DiskPressure condition"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesOutOfDisk
+      expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes out of disk"
+        description: "{{ $labels.node }} has is out of disk space"
+        dashboard: "Check GKE or EKS node status"
+
+    - alert: KubernetesVolumeOutOfDiskSpace
+      expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
+      for: 5m
+      labels:
+        severity: error
+        priority: P2
+      annotations:
+        summary: "Kubernetes PVC Free Space < 10%"
+        description: "PVC volume {{ $labels.persistentvolumeclaim }} is almost full - down to {{ $value | humanize }}% available"
+        dashboard: "OVH:<MONITORING>/k8s-pvc/pvc-usage?var-volume={{ $labels.persistentvolumeclaim }}"
+
+#   - alert: KubernetesVolumeFullInFourDays
+#     expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0
+#     for: 5m
+#     labels:
+#       severity: error
+#     annotations:
+#       summary: "Kubernetes Volume full in four days "
+#       description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available."
+#       dashboard: "OVH:<MONITORING>/pvc"
+
+    - alert: KubernetesPersistentvolumeError
+      expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes PersistentVolume error "
+        description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is stuck pending"
+        dashboard: "Check GKE or EKS PVC/storage status"
+
+    - alert: KubernetesHpaScaleCapability
+      expr: kube_hpa_status_desired_replicas >= kube_hpa_spec_max_replicas
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Kubernetes HPA scale capability"
+        description: "Kubernetes has scaled {{ $labels.hpa }} to max number of scaled pods."
+        dashboard: "OVH:<MONITORING>/k8-deployment"
+
+    - alert: KubernetesPodNotHealthy
+      expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed"} == 1
+      for: 20m
+      labels:
+        severity: error
+      annotations:
+        summary: "Kubernetes Pod not healthy"
+        description: "Pod {{ $labels.pod }} has been in a non-ready state for longer than 20 minutes."
+        dashboard: "Check the pod status using the Kubernetes UI"
+
+    - alert: KubernetesPodCrashLooping
+      expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5
+      for: 5m
+      labels:
+        severity: page
+        priority: P2
+      annotations:
+        summary: "Kubernetes pod crash looping"
+        description: "{{ $labels.pod }} is in a crash loop"
+        dashboard: "Check the pod status using the Kubernetes UI"

package/src/helm-charts/prom-operator/rules/leverege-rules.yaml

@@ -0,0 +1,20 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: leverege-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Leverege
+    rules:
+    - alert: frequent_app_restarts
+      expr: rate(nodejs_app_exit_count[15m]) * 60 * 5 > 5
+      for: 1m
+      labels:
+        severity: page
+        priority: P2
+      annotations:
+        summary: Apps are restarting frequently
+        description: "{{ $labels.app }} is in a crash loop"
+        dashboard: "OVH:<MONITORING>/stackdriver"

package/src/helm-charts/prom-operator/rules/postgres-rules.yaml

@@ -0,0 +1,190 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: postgres-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Postgres
+    rules:
+    - alert: PostgresqlDown
+      expr: pg_up == 0
+      for: 5m
+      labels:
+        severity: page
+        priority: P3
+      annotations:
+        summary: "Postgresql down"
+        description: "Postgresql instance is down: check {{ $labels.release }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlRestarted
+      expr: time() - pg_postmaster_start_time_seconds < 60
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql restarted"
+        description: "Postgresql restarted: check {{ $labels.release }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlExporterError
+      expr: pg_exporter_last_scrape_error > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql exporter error for {{ $labels.release }}"
+        description: "Postgresql exporter is showing errors. Metrics may be outdated"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlReplicationLag
+      expr: (pg_replication_lag > 10) and ON(instance) (pg_replication_is_replica == 1)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql replication lag"
+        description: "PostgreSQL replication lag is going up (> 10s): check {{ $labels.release }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTableNotVacuumed
+      expr: (pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql table not vacuumed"
+        description: "Table has not been vacuum for 24 hours: check {{ $labels.instance }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTableNotAutoAnalyzed
+      expr: (pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql table not analyzed (instance {{ $labels.instance }})"
+        description: "Table {{ $labels.relname }} has not been analyzed for 24 hours: check {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlDeadLocks
+      expr: rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql dead locks"
+        description: "PostgreSQL has dead-locks: check {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlSlowQueries
+      expr: pg_slow_queries > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql slow queries"
+        description: "PostgreSQL executing slow queries"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlHighRollbackRate
+      expr: rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql high rollback rate on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Ratio of transactions being aborted compared to committed is > 2 %"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlCommitRateLow
+      expr: rate(pg_stat_database_xact_commit[1m]) < 10
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql commit rate low on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Postgres seems to be processing very few transactions"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlLowXidConsumption
+      expr: rate(pg_txid_current[1m]) < 5
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql low XID consumption on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Postgresql seems to be consuming transaction IDs very slowly"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqllowXlogConsumption
+      expr: rate(pg_xlog_position_bytes[1m]) < 100
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresqllow XLOG consumption on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "Postgres seems to be consuming XLOG very slowly"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlWaleReplicationStopped
+      expr: rate(pg_xlog_position_bytes[1m]) == 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql WAL-E replication stopped on {{ $labels.release }} - {{ $labels.datname }}"
+        description: "WAL-E replication seems to be stopped"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlHighRateStatementTimeout
+      expr: rate(postgresql_errors_total{type="statement_timeout"}[5m]) > 3
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql high rate statement timeout"
+        description: "Postgres transactions showing high rate of statement timeouts on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlHighRateDeadlock
+      expr: rate(postgresql_errors_total{type="deadlock_detected"}[1m]) * 60 > 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql high rate deadlock"
+        description: "Postgres detected deadlocks on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlReplicationLabBytes
+      expr: (pg_xlog_position_bytes and pg_replication_is_replica == 0) - on (environment) group_right(instance) (pg_xlog_position_bytes and pg_replication_is_replica == 1) > 1e+09
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql replication lab bytes"
+        description: "Postgres Replication lag (in bytes) is high on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTooManyDeadTuples
+      expr: ((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Postgresql too many dead tuples"
+        description: "PostgreSQL dead tuples is too large ( {{ $value }} dead tuples) on {{ $labels.release }} - {{ $labels.datname }}"
+        dashboard: "OVH:<MONITORING>/postgres"
+
+    - alert: PostgresqlTooManyLocksAcquired
+      expr: ((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Postgresql too many locks acquired on {{ $labels.statefulset.kubernetes.io_pod_name }}"
+        description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction."
+        dashboard: "OVH:<MONITORING>/postgres"

package/src/helm-charts/prom-operator/rules/prometheus-rules.yaml

@@ -0,0 +1,79 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Prometheus
+    rules: 
+    - alert: targets_down
+      expr: up{target_env!="preemptibles"} == 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: Prometheus target missing
+        description: "Prometheus failed to scrape {{ $labels.app }}"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusConfigurationReloadFailure
+      expr: prometheus_config_last_reload_successful != 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus server configuration reload failure"
+        description: "Prometheus server failed to reload server configs (e.g. alerts, rules)"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusAlertmanagerConfigurationReloadFailure
+      expr: alertmanager_config_last_reload_successful != 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus AlertManager configuration reload failure"
+        description: "AlertManager failed to reload alert configs (e.g. slack, opsgenie)"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusTooManyRestarts
+      expr: changes(process_start_time_seconds{job=~"prometheus|alertmanager"}[15m]) > 2
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus is in a crash loop"
+        description: "Prometheus {{ $labels.component }} has restarted more than twice in the last 15 minutes. It might be crashlooping."
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusNotConnectedToAlertmanager
+      expr: prometheus_notifications_alertmanagers_discovered < 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Prometheus not connected to alertmanager"
+        description: "Prometheus cannot connect the alertmanager - new alerts may not be detected"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusNotificationsBacklog
+      expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Prometheus notifications backlog"
+        description: "The Prometheus notification queue has not been empty for 10 minutes"
+        dashboard: "See prometheus.md in SRE Playbook"
+
+    - alert: PrometheusAlertmanagerNotificationFailing
+      expr: rate(alertmanager_notifications_failed_total[1m]) > 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Prometheus AlertManager notification failing"
+        description: "Alertmanager is failing to send notifications"
+        dashboard: "See prometheus.md in SRE Playbook"

package/src/helm-charts/prom-operator/rules/redis-rules.yaml

@@ -0,0 +1,90 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: redis-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Redis
+    rules:             
+    - alert: RedisDown
+      expr: redis_up == 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis down (instance {{ $labels.instance }})"
+        description: "Redis instance is down"
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisMissingMaster
+      expr: count(redis_instance_info{role="master"}) == 0
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis missing master (instance {{ $labels.instance }})"
+        description: "Redis cluster has no node marked as master."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisTooManyMasters
+      expr: count(redis_instance_info{role="master"}) > 1
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis too many masters (instance {{ $labels.instance }})"
+        description: "Redis cluster has too many nodes marked as master."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisDisconnectedSlaves
+      expr: count without (instance, job) (redis_connected_slaves) - sum without (instance, job) (redis_connected_slaves) - 1 > 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis disconnected slaves (instance {{ $labels.instance }})"
+        description: "Redis not replicating for all slaves. Consider reviewing the redis replication status."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisReplicationBroken
+      expr: delta(redis_connected_slaves[1m]) < 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis replication broken (instance {{ $labels.instance }})"
+        description: "Redis instance lost a slave"
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisClusterFlapping
+      expr: changes(redis_connected_slaves[5m]) > 2
+      for: 5m
+      labels:
+        severity: error
+      annotations:
+        summary: "Redis cluster flapping (instance {{ $labels.instance }})"
+        description: "Changes have been detected in Redis replica connection. This can occur when replica nodes lose connection to the master and reconnect (a.k.a flapping)."
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisOutOfMemory
+      expr: redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis out of memory (instance {{ $labels.instance }})"
+        description: "Redis is running out of memory ( {{ $value }}% )"
+        dashboard: "OVH:<MONITORING>/redis"
+
+    - alert: RedisRejectedConnections
+      expr: increase(redis_rejected_connections_total[1m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Redis rejected connections (instance {{ $labels.instance }})"
+        description: "Some connections to Redis has been rejected. Please check {{ $labels.app }}"
+        dashboard: "OVH:<MONITORING>/redis"
+

package/src/helm-charts/prom-operator/rules/test-alert.yaml

@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: test-alert
+  namespace: prometheus  # Ensure this matches your Prometheus namespace
+spec:
+  groups:
+    - name: test.rules
+      rules:
+        - alert: TestAlert
+          expr: vector(1)  # This will always be true, triggering the alert
+          for: 1m  # Alert if condition is true for 1 minute
+          labels:
+            severity: warning
+          annotations:
+            summary: "Test Alert for Slack"
+            description: "This is a test alert to verify the Slack receiver configuration."
+

package/src/helm-charts/prom-operator/rules/traefik-rules.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: traefik-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Traefik Alerts
+    rules:
+    - alert: TraefikConfigReloadFailed
+      expr: traefik_config_last_reload_failure > 0
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Traefik config failed to reload"
+        description: "New config file for Traefik is invalid or Traefik encountered an error"
+
+#   - alert: TraefikHighHttp4xxError
+#     expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
+#     for: 5m
+#     labels:
+#       severity: error
+#     annotations:
+#       summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
+#       description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
+#       dashboard: "OVH:<MONITORING>/traefik"

package/src/helm-charts/prom-operator/rules/velero-rules.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: velero-rules
+  namespace: prometheus
+
+spec:
+  groups:
+  - name: Velero
+    rules:
+    - alert: VeleroBackupFailure
+      expr: (time() - velero_backup_last_successful_timestamp{schedule_version=~"2.[0]"} > 108000) and (velero_backup_success_total{schedule!=""}  != 0)
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Velero schedule {{ $labels.schedule }} backup failed to execute properly"
+        description: "It has been over 30 hours since the last successful backup"
+        dashboard: "OVH:<MONITORING>/velero"
+    - alert: NoVeleroBackups
+      expr: velero_backup_success_total{schedule!=""}  == 0
+      for: 30h
+      labels:
+        severity: warning
+      annotations:
+        summary: "Velero has not made any backups yet"
+        description: "No backups have successfully been made since velero was initialized (or restarted), go check it out ASAP"
+        dashboard: "OVH:<MONITORING>/velero"

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -0,0 +1,57 @@
+#
+# to force a reinstall of this service simply remove or rename this
+# stackdriver-exporter subdirectory and rerun helmup stackdriver-exporter
+#
+fullnameOverride: "prometheus-stack-stackdriver-metrics"
+
+stackdriver:
+  projectId: "OVH:<PROJECT_NAME>"
+  metrics:
+    typePrefixes: "pubsub.googleapis.com/subscription/oldest_unacked_message_age,pubsub.googleapis.com/subscription/num_undelivered_messages,pubsub.googleapis.com/subscription/pull_ack_request_count,pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,pubsub.googleapis.com/topic/message_sizes,pubsub.googleapis.com/topic/send_request_count,firebasedatabase.googleapis.com/io/database_load,firebasedatabase.googleapis.com/network/sent_bytes_count,firebasedatabase.googleapis.com/network/active_connections"
+
+# https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter?modal=values&path=serviceMonitor
+serviceMonitor:
+  enabled: true
+  namespace: prometheus
+  interval: "30s"
+  relabelings:
+    - action: labelmap
+      regex: __meta_kubernetes_pod_label_(.+)
+    - action: labelmap
+      regex: __meta_kubernetes_service_label_(.+)
+
+annotations:
+  prometheus.io/port: "9255"
+  prometheus.io/scrape: "true" # this is a string annotation - so quoted!
+
+tolerations:
+- key: "preemptible"
+  operator: "Equal"
+  value: "true"
+  effect: "NoSchedule"
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: target-env
+          operator: In
+          values:
+          - preemptible
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 32Mi
+
+# NOTE to future selves - the exporter's helm chart does not have a clue
+# regarding workload identity, which means the gcloud and k8s service
+# accounts have to be managed outside of the chart by helmup. In order to get
+# the linkages from the SA to the service we specify the SA name here, but
+# disable the creation of the SA since it will have already have been created
+# and bound by the helmup installStackdriverExporterEnvironment function.
+#
+serviceAccount:
+  create : false # this is a flag to helm - so unquoted ... da F?!
+  name: "stackdriver-exporter"

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.1.7"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.2.1"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/helm-charts/redis/redis-local.yaml

@@ -70,3 +70,11 @@ networkPolicy:
 
 metrics:
   enabled: true
+  serviceMonitor:
+    enabled: false # set to true for prometheus operator
+    namespace: "prometheus"
+    relabelings:
+    - action: labelmap
+      regex: __meta_kubernetes_pod_label_(.+)
+    - action: labelmap
+      regex: __meta_kubernetes_service_label_(.+)

package/src/helm-charts/traefik/helmup.plugin

@@ -11,13 +11,13 @@
 # kubectl apply --namespace traefik --server-side \
 #               --force-conflicts -k https://github.com/traefik/traefik-helm-chart/traefik/crds/\?ref\=v27
 #
-showInstalling "Traefik Load Balancer / Router"
+showInstalling "Traefik v3 Load Balancer / Router"
 
 TRAEFIK_NAMESPACE="traefik"
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="27"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="32"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/traefik-local.yaml \

package/src/helmup.sh

@@ -296,7 +296,7 @@ function installStackdriverExporterEnvironment() {
   # Add necessary helm repositories - note that stable is deprecated
   addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-  local SDEXP_NS="monitoring"
+  local SDEXP_NS=${1:-monitoring}
   local SDEXP_SA="stackdriver-exporter"
   local SDEXP_EM="$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
 
@@ -306,7 +306,8 @@ function installStackdriverExporterEnvironment() {
   printf "\n*** Removing the previous $SDEXP_SA installation...\n"
   helm uninstall -n $SDEXP_NS $SDEXP_SA                 &> $DEVNULL
   kubectl delete serviceaccounts -n $SDEXP_NS $SDEXP_SA &> $DEVNULL
-  gcloud --quiet iam service-accounts delete $SDEXP_EM  &> $DEVNULL
+  printf "!!! Skipping deletion of the gcloud SA $SDEXP_EM\n"
+# gcloud --quiet iam service-accounts delete $SDEXP_EM  &> $DEVNULL
 
   # NOTE: The k8s SA != GCP SA - the former is created by the helm chart
   # that installs the stackdriver-exporter, the latter is handled here in
@@ -321,7 +322,7 @@ function installStackdriverExporterEnvironment() {
 
   printf "\n*** Binding IAM role of $SDEXP_NS viewer to $SDEXP_SA in $GCP_PROJECT_ID\n"
   gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \
-    --role "roles/$SDEXP_NS.viewer" \
+    --role "roles/monitoring.viewer" \
     --member "serviceAccount:$SDEXP_EM" &> $DEVNULL
   sleep 2 # give the IAM binding a chance to complete before moving on
 
@@ -1002,7 +1003,13 @@ CATBACKUP
         bootstrapLocalSetup $SERVICE
         ;;
 
-      "prometheus"|"prometheus-operator")
+      "prometheus")
+        installStackdriverExporterEnvironment
+        bootstrapLocalSetup $SERVICE
+        ;;
+
+      "prom-operator")
+        installStackdriverExporterEnvironment prometheus
         bootstrapLocalSetup $SERVICE
         ;;
 

package/src/overwhelm.mjs

@@ -206,6 +206,13 @@ const doReplacements = ( yamls, replacements ) => {
     process.exit( 1 )
   }
 
+  // Eat any doubly double quoted values such that
+  //   ""<ovh replace>"" -> "<ovh replace>"
+  // but
+  //   someDefault: ""
+  // remains unchanged
+  replYaml = replYaml.replace( /""([^"]+)""/g, '"$1"' )
+
   return replYaml
 }
 
@@ -307,7 +314,7 @@ ahoy.forEach( ( dir ) => {
   }
 } )
 
-// we can also apply replaceables to any top level .ovh files to support top level config maps
+// we can also apply replaceables to all .ovh files in the project
 glob.sync( '**/*.ovh' ).forEach( ( cfgmap ) => {
   const target = cfgmap.replace( /.ovh$/, '' )
   let valuesOut = fs.readFileSync( cfgmap, { encoding : 'utf-8' } )