@leverege/build-tools 2.66.2 → 2.66.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.66.2",
+  "version": "2.66.4",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -69,10 +69,10 @@
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
     "execa": "^9.5.2",
-    "glob": "^11.0.1",
+    "glob": "^11.0.2",
     "handlebars": "^4.7.8",
-    "ignore": "^7.0.3",
-    "inquirer": "^12.5.2",
+    "ignore": "^7.0.4",
+    "inquirer": "^12.6.0",
     "js-yaml": "^4.1.0",
     "jsdoc": "^4.0.4",
     "ms": "^2.1.3",

package/src/bash-funcs

@@ -215,3 +215,13 @@ PLUGIN_NOEXEC
   sendToSlack "$PLUGIN"
   return 0
 }
+
+# simple enforcement of necessary tools that need to be installed
+function commandChecker() {
+  for cmd in "$@"; do
+    command -v "$cmd" &> $DEVNULL
+    exitOnError $? "the `color g $cmd` command is required but not installed please install it and try again"
+  done
+}
+
+commandChecker git helm kubectl service-man

package/src/helm-audit.mjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+
+// helm-audit.mjs
+// Node.js version of helm-audit.sh to display deployed chart/image/audit info
+
+import { shellCmd, log, debug } from './Utils.mjs'
+import ora from 'ora'
+import pLimit from 'p-limit'
+import chalk from 'chalk'
+
+const spinner = ora('Gathering Helm release info...').start()
+const limit = pLimit(5) // limit concurrent shellCmds to avoid API overload
+const OMIT_NAMESPACES = new Set(['estafette'])
+
+async function getHelmReleases() {
+  const json = await shellCmd('helm list -A -o json')
+  return JSON.parse(json).filter(r => !OMIT_NAMESPACES.has(r.namespace))
+}
+
+async function getDeployedImageTag(ns, release) {
+  try {
+    const image = await shellCmd(`kubectl get deployment -n ${ns} -l app.kubernetes.io/instance=${release} -o jsonpath={.items[0].spec.template.spec.containers[0].image}`)
+    const tag = image.trim().split(':')[1]
+    return tag && tag !== 'not found' ? tag : ''
+  } catch {
+    return ''
+  }
+}
+
+async function getChartAppVersion(ns, release) {
+  try {
+    const metadata = await shellCmd(`helm get metadata ${release} -n ${ns}`)
+    const match = metadata.match(/^APP_VERSION:\s*(.*?)\s*$/m)
+    return match ? match[1].trim() : '(unknown)'
+  } catch {
+    return '(unknown)'
+  }
+}
+
+async function getHelmValue(ns, release, key) {
+  try {
+    const values = await shellCmd(`helm get values ${release} -n ${ns} -o json`)
+    const parsed = JSON.parse(values)
+    return parsed.helmup?.[key] || ''
+  } catch {
+    return ''
+  }
+}
+
+function normalizeTag(tag) {
+  return tag?.replace(/^v/, '').toLowerCase() || ''
+}
+
+function formatRow(name, chartVersion, appVersion, imageTag, deployT, culprit) {
+  const release = chalk.green(name.slice(0, 25).padEnd(32))
+  const chartVer = chalk.cyan(chartVersion.padEnd(18))
+  const appVer = chalk.cyan(appVersion.padEnd(18))
+  const normApp = normalizeTag(appVersion)
+  const normImage = normalizeTag(imageTag)
+
+  let imageFormatted = imageTag.padEnd(20)
+  if (normImage && normApp && normImage === normApp) {
+    imageFormatted = chalk.green(imageFormatted)
+  } else if (imageTag) {
+    imageFormatted = chalk.red(imageFormatted)
+  }
+
+  const deployInfo = deployT ? chalk.gray(`${deployT} by ${culprit || '(unknown)'}`) : ''
+  return `  ${release}${chartVer}${appVer}${imageFormatted}${deployInfo}`
+}
+
+async function audit() {
+  spinner.text = 'Fetching release metadata...'
+  const releases = await getHelmReleases()
+
+  // sort by namespace, then release name
+  releases.sort((a, b) => {
+    const nsCompare = a.namespace.localeCompare(b.namespace)
+    return nsCompare !== 0 ? nsCompare : a.name.localeCompare(b.name)
+  })
+
+  spinner.text = 'Gathering chart, image, and audit metadata...'
+
+  const rows = await Promise.all(releases.map(({ name, namespace, chart }) => limit(async () => {
+    const chartVersion = chart?.split('-').pop() || '(unknown)'
+    const [appVersion, imageTag, deployT, culprit] = await Promise.all([
+      getChartAppVersion(namespace, name),
+      getDeployedImageTag(namespace, name),
+      getHelmValue(namespace, name, 'deployT'),
+      getHelmValue(namespace, name, 'culprit'),
+    ])
+    return {
+      row: formatRow(name, chartVersion, appVersion, imageTag, deployT, culprit),
+      namespace
+    }
+  })))
+
+  spinner.stop()
+  log(`${'RELEASE'.padEnd(34)}${'CHART VERSION'.padEnd(18)}${'APP VERSION'.padEnd(18)}${'IMAGE TAG'.padEnd(20)}DEPLOY INFO`)
+
+  let lastNamespace = ''
+  for (const { row, namespace } of rows) {
+    if (namespace !== lastNamespace) {
+      if (lastNamespace !== '') log('')
+      log(chalk.bold.green(namespace))
+      lastNamespace = namespace
+    }
+    log(row)
+  }
+}
+
+audit().catch(err => {
+  spinner.stop()
+  console.error(err)
+  process.exit(1)
+})

package/src/helm-charts/api-server/values-local.yaml

@@ -5,7 +5,7 @@
 #     - node tools/UpdateESMappingV2.js
 #
 image:
-  tag: # v5.17.5
+  tag: # v5.25.4
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
@@ -28,10 +28,13 @@ psqlSecurity:
   TSDB_DENSE_SECRET_NAME: "cnpg-db-tsdb-dense-postgres-pw"
   TSDB_DENSE_SECRET_KEY: "password"
 
+serviceMonitor:
+  enabled: true
+
 autoscaling:
   minReplicas: 3
   maxReplicas: 12
   maxCPU: 80
 
 ingressRoute:
-  host: OVH:<PROJECT_ID>-imagine-api.OVH:<HOST>.com
+  host: ${PROJECT_ID}-imagine-api.${HOST}.com

package/src/helm-charts/authz-server/values-local.yaml

@@ -4,10 +4,13 @@
 #     - npm run sequelizeRunMigrations
 #
 image:
-  tag: # v2.4.0
+  tag: # v2.5.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
   # CNPG override
   SQL_HOST: "cnpg-db-psql-stack-rw.cnpg-operands" # vs postgres-postgresql
+
+serviceMonitor:
+  enabled: true

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -18,7 +18,7 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
-        - CREATE DATABASE pg_stat_statements;
+        - CREATE EXTENSION pg_stat_statements;
         - CREATE DATABASE imagine;
 #       - CREATE DATABASE app; # for the operator restores
 #       - CREATE ROLE app;     # same

package/src/helm-charts/db-curator/values-local.yaml

@@ -1,7 +1,7 @@
 image:
-  tag: # v4.4.2
+  tag: # v4.6.2
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
   FIREBASE_CLEANUP_ENABLED: "true"
-  TSDB_CLEANUP_ENABLED: "false" # no need is new tsdb using autoexpiry
+  TSDB_CLEANUP_ENABLED: "false"

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,7 +2,8 @@
 #
 showInstalling "Elastic Search 8"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.6.0"
+# *** WARNING TO FUTURE SELF *** v22 of the chart is ES v9!!!
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.6.3"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 

package/src/helm-charts/emailer/values-local.yaml

@@ -4,6 +4,9 @@ image:
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
-  API_UI_URL: "OVH:<PROJECT_ID>.web.app"
-  MAIL_GUN_DOMAIN: "OVH:<MAIL_GUN_DOMAIN>"
-  MAIL_GUN_DEFAULT_SENDER: "no-reply@OVH:<MAIL_GUN_DOMAIN>"
+  API_UI_URL: "${PROJECT_ID}.web.app"
+  MAIL_GUN_DOMAIN: "${MAIL_GUN_DOMAIN}"
+  MAIL_GUN_DEFAULT_SENDER: "no-reply@${MAIL_GUN_DOMAIN}"
+
+serviceMonitor:
+  enabled: true

package/src/helm-charts/fota-server/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v2.1.0
+  tag: # v2.1.3
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/geotile-server/values-local.yaml

@@ -2,6 +2,9 @@ image:
   tag: # v2.3.0
 
 config:
-  UPLOADER_TYPE: "OVH:<BUCKET_TYPE>"
-  UPLOADER_PROJECT_ID: "OVH:<PROJECT_ID>"
-  UPLOADER_BUCKET: "OVH:<PROJECT_ID>-geotile-server"
+  UPLOADER_TYPE: "${BUCKET_TYPE}"
+  UPLOADER_PROJECT_ID: "${PROJECT_ID}"
+  UPLOADER_BUCKET: "${PROJECT_ID}-geotile-server"
+
+serviceMonitor:
+  enabled: true

package/src/helm-charts/imagine/values-local.yaml

@@ -3,3 +3,6 @@ mage:
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
+
+serviceMonitor:
+  enabled: true

package/src/helm-charts/message-processor/values-local.yaml

@@ -1,8 +1,8 @@
 image:
-  tag: # v4.2.0
+  tag: # v4.3.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
 serviceMonitor:
-  enabled: false
+  enabled: true

package/src/helm-charts/messenger/values-local.yaml

@@ -1,6 +1,9 @@
 image:
- tag: # v2.14.0
+ tag: # v2.16.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
-  API_UI_URL: "OVH:<PROJECT_ID>.web.app"
+  API_UI_URL: "${PROJECT_ID}.web.app"
+
+serviceMonitor:
+  enabled: true

package/src/helm-charts/overdose/values-local.yaml

@@ -17,3 +17,6 @@ autoscaling:
   minReplicas: 6
   maxReplicas: 16
   averageCPU: 75
+
+serviceMonitor:
+  enabled: true

package/src/helm-charts/prom-operator/helmup.plugin

@@ -3,28 +3,35 @@
 showInstalling "The Prometheus Operator and Components"
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-showInstalling "The Prometheus Operator (kube-prometheus-stack)"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="70"
-
 NS="--namespace prometheus"
 
+showInstalling "The Prometheus Operator (kube-prometheus-stack)"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="72"
+
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 helm upgrade $NS --install prometheus-stack prometheus-community/kube-prometheus-stack \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-elasticsearch-exporter
-showInstalling "The Elasticsearch Exporter (prom-operator)"
+showInstalling "The Elasticsearch Exporter (prom-operator ES metrics exports)"
 [ -z "$ELASTICSEARCH_EXPORTER_CHART_VERSION" ] && ELASTICSEARCH_EXPORTER_CHART_VERSION="6"
 helm upgrade $NS --install elasticsearch8-exporter prometheus-community/prometheus-elasticsearch-exporter \
                  --values prom-operator/elasticsearch-exporter.yaml \
                  --version $ELASTICSEARCH_EXPORTER_CHART_VERSION $HELM_WHAT
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter
-showInstalling "The Stackdriver Exporter (prom-operator)"
+showInstalling "The Stackdriver Exporter (prom-operator GKE metrics exports)"
 [ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="4"
 helm upgrade $NS --install stackdriver-exporter prometheus-community/prometheus-stackdriver-exporter \
                  --values prom-operator/stackdriver-exporter.yaml \
                  --version $STACKDRIVER_EXPORTER_CHART_VERSION $HELM_WHAT
 
+# https://artifacthub.io/packages/helm/prometheus-community/prometheus-adapter
+showInstalling "The Prometheus Adapter (prom-operator custom HPA metrics)"
+[ -z "$PROMETHEUS_ADAPTER_CHART_VERSION" ] && PROMETHEUS_ADAPTER_CHART_VERSION="4"
+helm upgrade $NS --install prometheus-adapter prometheus-community/prometheus-adapter \
+                 --values prom-operator/prometheus-adapter.yaml \
+                 --version $PROMETHEUS_ADAPTER_CHART_VERSION $HELM_WHAT
+
 removeHelmRepo prometheus-community

package/src/helm-charts/prom-operator/prometheus-adapter.yaml

@@ -0,0 +1,18 @@
+fullnameOverride: "prometheus-stack-adapter"
+
+prometheus:
+  url: "http://prometheus-operated.prometheus.svc"
+  port: 9090
+
+rules:
+  custom:
+    - seriesQuery: 'gpu_duty_cycle_percent{namespace!="",pod!=""}'
+      resources:
+        overrides:
+          namespace: {resource: "namespace"}
+          pod: {resource: "pod"}
+      name:
+        matches: "gpu_duty_cycle_percent"
+        as: "gpu_duty_cycle_percent"
+      metricsQuery: |
+        avg(gpu_duty_cycle_percent{<<.LabelMatchers>>}) by (<<.GroupBy>>)

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -82,7 +82,7 @@ grafana:
         entryPoints:
           - websecure
         routes:
-          - match: Host(`${PROJECT_NAME}-promop.${HOST}.com`)
+          - match: Host(`${PROJECT_NAME}-monitoring.${HOST}.com`)
             kind: Rule
             services:
             - name: prometheus-stack-grafana
@@ -114,16 +114,57 @@ alertmanager:
         slack_configs:
           - channel: "${SLACK_CHANNEL}"
             api_url: "${SLACK_HOOK_URL}"
-            username: "${PROJECT_NAME}"
-            fallback: "${PROJECT_NAME} - {{ .CommonAnnotations.summary }}"
-            title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"
-            title_link: "https://bitbucket.org/leverege/sre-playbook/src/master/"
+            username: "${PROJECT_ID}"
             send_resolved: true
+            title: '{{ .CommonLabels.alertname }} alert'
+            title_link: '#'
+            color: >-
+              {{- if eq .Status "resolved" -}}
+                good
+              {{- else if eq .CommonLabels.severity "critical" -}}
+                danger
+              {{- else if eq .CommonLabels.severity "warning" -}}
+                warning
+              {{- else if eq .CommonLabels.severity "info" -}}
+                #439FE0
+              {{- else -}}
+                #CCCCCC
+              {{- end }}
             text: |-
               {{ range .Alerts }}
-                *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
-                *Description:* {{ .Annotations.description }}
-                *Dashboard:* {{ .Annotations.dashboard }}
+              {{- if eq .Status "resolved" }}
+                :white_check_mark: *Resolved:* {{ .Annotations.summary }}
+              {{- else if eq .Labels.severity "critical" }}
+                :fire: *Critical:* {{ .Annotations.summary }}
+              {{- else if eq .Labels.severity "warning" }}
+                :warning: *Warning:* {{ .Annotations.summary }}
+              {{- else if eq .Labels.severity "info" }}
+                :information_source: *Info:* {{ .Annotations.summary }}
+              {{- else }}
+                :grey_question: *Alert:* {{ .Annotations.summary }}
+              {{- end }}
+
+              *Description:* {{ .Annotations.description }}
+
+              {{- if .Annotations.runbook_url }}
+              *Runbook:* <{{ .Annotations.runbook_url }}|Open Playbook>
+              {{- end }}
+
+              {{- if .Labels.pod }}
+              *Pod Link:* *<https://console.cloud.google.com/kubernetes/pod/${GCE_REGION}/${CLUSTER_NAME}/{{ .Labels.namespace }}/{{ .Labels.pod }}?project=${PROJECT_ID}|`{{ .Labels.pod }}`>*
+              {{- end }}
+
+              *Status:* `{{ .Status }}` | *Severity:* `{{ .Labels.severity }}` {{- if .Labels.container }} *Container:* `{{ .Labels.container }}` {{- end }}
+  
+              {{- if .Annotations.dashboard }}
+              *Dashboard:* <{{ .Annotations.dashboard }}|View Dashboard>
+              {{- end }}
+
+              *Labels:*
+              {{- range $k, $v := .Labels }}
+                • `{{ $k }} = {{ $v }}`
+              {{- end }}
+              ---
               {{ end }}
       - name: slack-watchdog
         slack_configs:
@@ -156,10 +197,10 @@ alertmanager:
       group_interval: 5m
       group_by:
         - alertname
-        - cluster
+        - namespace
       receiver: slack
       routes:
-      - match:
+      - match_re:
           alertname: "Watchdog"
         receiver: slack-watchdog
       - match:

package/src/helm-charts/prom-operator/rules/cadvisor-rules.yaml

@@ -64,16 +64,31 @@ spec:
                 VALUE = {{ $value }}
                 LABELS = {{ $labels }}
         - alert: ContainerHighThrottleRate
-          expr: sum(increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) by (container, pod, namespace) / sum(increase(container_cpu_cfs_periods_total[5m])) by (container, pod, namespace) > ( 25 / 100 )
+          expr: |
+            (
+              (
+                sum by (container, pod, namespace) (increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m]))
+                /
+                sum by (container, pod, namespace) (increase(container_cpu_cfs_periods_total{container!=""}[5m]))
+              )
+              and on (pod, namespace)
+              (
+                time() - kube_pod_start_time > 60
+              )
+            ) > 0.3
           for: 5m
           labels:
             severity: warning
+            namespace: '{{ $labels.namespace }}'
+            pod: '{{ $labels.pod }}'
+            container: '{{ $labels.container }}'
           annotations:
-            summary: Container high throttle rate (instance {{ $labels.instance }})
-            description: |-
-              Container is being throttled
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+            summary: Pod experiencing high CPU throttling
+            description: |
+              {{ $labels.pod }} was throttled
+              - Verify Kubernetes CPU resource *requests* and *limits* for the container.
+              - Check if recent pod startup or autoscaling events could explain transient throttling.
+              - Persistent throttling indicates the container needs higher CPU limits or node sizing adjustment.
 #       - alert: ContainerHighLowChangeCpuUsage
 #         expr: (abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m] offset 1m)) * 100)) or abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[5m] offset 1m)) * 100))) > 25
 #         for: 0m

package/src/helm-charts/prom-operator/rules/elasticsearch-rules.yaml

@@ -174,7 +174,7 @@ spec:
                 VALUE = {{ $value }}
                 LABELS = {{ $labels }}
         - alert: ElasticsearchHighIndexingLatency
-          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.0005
+          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.002 # awesome was 0.0005
           for: 10m
           labels:
             severity: warning

package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml

@@ -172,8 +172,8 @@ spec:
             summary: Kubernetes HPA scale inability (instance {{ $labels.instance }})
             description: |-
               HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to scale
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+#               VALUE = {{ $value }}
+#               LABELS = {{ $labels }}
         - alert: KubernetesHpaMetricsUnavailability
           expr: kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive"} == 1
           for: 0m
@@ -183,8 +183,8 @@ spec:
             summary: Kubernetes HPA metrics unavailability (instance {{ $labels.instance }})
             description: |-
               HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to collect metrics
-                VALUE = {{ $value }}
                 LABELS = {{ $labels }}
+#               VALUE = {{ $value }}
         - alert: KubernetesHpaScaleMaximum
           expr: (kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas) and (kube_horizontalpodautoscaler_spec_max_replicas > 1) and (kube_horizontalpodautoscaler_spec_min_replicas != kube_horizontalpodautoscaler_spec_max_replicas)
           for: 2m
@@ -193,9 +193,7 @@ spec:
           annotations:
             summary: Kubernetes HPA scale maximum (instance {{ $labels.instance }})
             description: |-
-              HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit maximum number of desired pods
-                VALUE = {{ $value }}
-                LABELS = {{ $labels }}
+              HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit max replica pods of {{ $value }}
         - alert: KubernetesHpaUnderutilized
           expr: max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3
           for: 0m
@@ -385,7 +383,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesApiServerLatency
           expr: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~"(?:CONNECT|WATCHLIST|WATCH|PROXY)"} [10m])) WITHOUT (subresource)) > 1
-          for: 2m
+          for: 15m
           labels:
             severity: warning
           annotations:

package/src/helm-charts/prom-operator/rules/prometheus-rules.yaml

@@ -21,7 +21,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: PrometheusTargetMissing
           expr: up{"namespace"!~"kube-system"} == 0 # awesome was => up == 0
-          for: 1m # awesome was => 0m
+          for: 3m # awesome was => 0m
           labels:
             severity: critical
           annotations:

package/src/helm-charts/prom-operator/stackdriver-exporter-gpu-rules.yaml

@@ -0,0 +1,44 @@
+#
+# these custom rules exist to essentially boil down the GPU information provided
+# by the stackdriver exporter into a form that is a little more humand readable
+# in an attempt to keep the grafana / alert manager rules a little less "verbose"
+#
+# In a nutshell, we take metrics in the form of:
+#   stackdriver_k_8_s_container_kubernetes_io_container_accelerator_duty_cycle
+# and create boiled down metrics in the form of:
+#   gpu_duty_cycle_percent
+#
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: prometheus-stack-kube-prom-stackdriver.gpu-rules
+  namespace: prometheus
+  labels:
+    release: prometheus-stack  # or whatever your Prometheus release name is
+spec:
+  groups:
+    - name: gpu.rules
+      interval: 1m
+      rules:
+        - record: "gpu_duty_cycle_percent"
+          expr: |
+            label_replace(
+              label_replace(
+                avg by (
+                  pod_name,
+                  namespace_name,
+                  container_name,
+                  accelerator_id,
+                  model,
+                  make
+                ) (
+                  stackdriver_k_8_s_container_kubernetes_io_container_accelerator_duty_cycle
+                ),
+                "namespace", "$1", "namespace_name", "(.*)"
+              ),
+              "pod", "$1", "pod_name", "(.*)"
+            )
+          labels:
+            source: "stackdriver"
+            metric_type: "derived"
+            subsystem: "gpu"

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -8,24 +8,27 @@ stackdriver:
   projectId: "${PROJECT_NAME}"
   metrics:
     typePrefixes:  >
-      pubsub.googleapis.com/subscription/oldest_unacked_message_age,
+      agent.googleapis.com/cpu/utilization,
+      agent.googleapis.com/memory/bytes_used,
+      container.googleapis.com/accelerator/duty_cycle,
+      container.googleapis.com/accelerator/memory_usage,
+      container.googleapis.com/accelerator/request_count,
+      firebasedatabase.googleapis.com/io/database_load,
+      firebasedatabase.googleapis.com/network/active_connections,
+      firebasedatabase.googleapis.com/network/sent_bytes_count,
+      kubernetes.io/container/accelerator/duty_cycle,
+      kubernetes.io/container/accelerator/memory_total,
+      kubernetes.io/container/accelerator/memory_used,
+      monitoring.googleapis.com/uptime_check/check_passed,
       pubsub.googleapis.com/subscription/num_undelivered_messages,
+      pubsub.googleapis.com/subscription/oldest_unacked_message_age,
       pubsub.googleapis.com/subscription/pull_ack_request_count,
       pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,
       pubsub.googleapis.com/topic/message_sizes,
       pubsub.googleapis.com/topic/send_request_count,
-      firebasedatabase.googleapis.com/io/database_load,
-      firebasedatabase.googleapis.com/network/sent_bytes_count,
-      firebasedatabase.googleapis.com/network/active_connections,
-      container.googleapis.com/accelerator/duty_cycle,
-      container.googleapis.com/accelerator/memory_usage,
-      container.googleapis.com/accelerator/request_count,
-      agent.googleapis.com/memory/bytes_used,
-      agent.googleapis.com/cpu/utilization,
+      quota.googleapis.com/allocation/usage,
       storage.googleapis.com/network/received_bytes_count,
-      storage.googleapis.com/network/sent_bytes_count,
-      monitoring.googleapis.com/uptime_check/check_passed,
-      quota.googleapis.com/allocation/usage
+      storage.googleapis.com/network/sent_bytes_count
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter?modal=values&path=serviceMonitor
 serviceMonitor:

package/src/helm-charts/pubsub-pulse/values-local.yaml

@@ -1,8 +1,10 @@
 image:
-  tag: # v1.6.3
-# registry: us-docker.pkg.dev/leverege-registry/leverege/images
+  tag: # v4.1.1
 
 config:
   pulseDelay: "1000"
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
-  IMAGINE_API_URL: "https://OVH:<PROJECT_ID>-imagine-api.OVH:<HOST>.com"
+  IMAGINE_API_URL: "https://${PROJECT_ID}-imagine-api.${HOST}.com"
+
+serviceMonitor:
+  enabled: true