@leverege/build-tools 2.124.0 → 2.124.1

package/src/dashboard/json/system/stackdriver.json

@@ -967,7 +967,7 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
           },
           "custom": {
             "axisBorderShow": false,
@@ -978,7 +978,7 @@
             "barAlignment": 0,
             "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "fillOpacity": 20,
+            "fillOpacity": 0,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
@@ -992,34 +992,48 @@
             "scaleDistribution": {
               "type": "linear"
             },
-            "showPoints": "never",
+            "showPoints": "auto",
             "spanNulls": false,
             "stacking": {
               "group": "A",
               "mode": "none"
             },
             "thresholdsStyle": {
-              "mode": "line+area"
+              "mode": "off"
             }
           },
           "mappings": [],
-          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
                 "color": "green",
                 "value": null
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Out-of-Order Samples (5m)"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "red",
+                  "mode": "fixed"
+                }
               },
               {
-                "color": "red",
-                "value": 1
+                "id": "custom.axisPlacement",
+                "value": "right"
               }
             ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
+          }
+        ]
       },
       "gridPos": {
         "h": 9,
@@ -1041,9 +1055,86 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "12.4.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "avg_over_time(scrape_duration_seconds{job=\"prometheus-stack-stackdriver-metrics\"}[5m])",
+          "legendFormat": "Scrape Duration (avg 5m)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(prometheus_tsdb_out_of_order_samples_total[5m]))",
+          "legendFormat": "Out-of-Order Samples (5m)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Stackdriver Scrape Duration vs TSDB Sample Rejections",
+      "type": "timeseries"
+    },
+    {
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 36
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.1.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "stackdriver_pubsub_subscription_pubsub_googleapis_com_subscription_dead_letter_message_count > 0",
+          "format": "time_series",
+          "legendFormat": "{{subscription_id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Dead Letter Message Count",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Bytes sent from GCS to clients, per bucket",
       "fieldConfig": {
         "defaults": {
-          "color": { "mode": "palette-classic" },
+          "color": {
+            "mode": "palette-classic"
+          },
           "custom": {
             "axisBorderShow": false,
             "axisCenteredZero": false,
@@ -1053,7 +1144,7 @@
             "barAlignment": 0,
             "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "fillOpacity": 0,
+            "fillOpacity": 20,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
@@ -1062,80 +1153,343 @@
             },
             "insertNulls": false,
             "lineInterpolation": "linear",
-            "lineWidth": 2,
+            "lineWidth": 1,
             "pointSize": 5,
-            "scaleDistribution": { "type": "linear" },
-            "showPoints": "auto",
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
             "spanNulls": false,
-            "stacking": { "group": "A", "mode": "none" },
-            "thresholdsStyle": { "mode": "off" }
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
-            "steps": [{ "color": "green", "value": null }]
-          }
-        },
-        "overrides": [
-          {
-            "matcher": { "id": "byName", "options": "Out-of-Order Samples (5m)" },
-            "properties": [
-              { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } },
-              { "id": "custom.axisPlacement", "value": "right" }
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
             ]
-          }
-        ]
+          },
+          "unit": "Bps",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 45
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "stackdriver_gcs_bucket_storage_googleapis_com_network_sent_bytes_count",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket_name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "GCS Network Egress",
+      "type": "timeseries"
+    },
+    {
       "datasource": {
         "type": "prometheus",
         "uid": "${datasource}"
       },
-      "pluginVersion": "12.4.2",
+      "description": "Bytes received into GCS from clients, per bucket",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "Bps",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 45
+      },
+      "id": 17,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
       "targets": [
         {
-          "datasource": { "type": "prometheus", "uid": "${datasource}" },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
           "editorMode": "code",
-          "expr": "avg_over_time(scrape_duration_seconds{job=\"prometheus-stack-stackdriver-metrics\"}[5m])",
-          "legendFormat": "Scrape Duration (avg 5m)",
+          "expr": "stackdriver_gcs_bucket_storage_googleapis_com_network_received_bytes_count",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket_name}}",
           "range": true,
           "refId": "A"
+        }
+      ],
+      "title": "GCS Network Ingress",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Number of objects stored per bucket",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "short",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 54
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
         {
-          "datasource": { "type": "prometheus", "uid": "${datasource}" },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
           "editorMode": "code",
-          "expr": "sum(increase(prometheus_tsdb_out_of_order_samples_total[5m]))",
-          "legendFormat": "Out-of-Order Samples (5m)",
+          "expr": "stackdriver_gcs_bucket_storage_googleapis_com_storage_object_count",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket_name}}",
           "range": true,
-          "refId": "B"
+          "refId": "A"
         }
       ],
-      "title": "Stackdriver Scrape Duration vs TSDB Sample Rejections",
+      "title": "GCS Object Count",
       "type": "timeseries"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Total bytes stored per bucket broken down by storage class (Standard, Nearline, Coldline, Archive). Requires storage.googleapis.com/storage/total_bytes prefix in stackdriver-exporter.yaml",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "bytes",
+          "unitScale": true
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 36
+        "y": 54
       },
-      "id": 14,
+      "id": 19,
       "options": {
         "legend": {
-          "calcs": [
-            "max"
-          ],
-          "displayMode": "table",
+          "calcs": [],
+          "displayMode": "list",
           "placement": "bottom",
           "showLegend": true
         },
         "tooltip": {
-          "hideZeros": false,
           "mode": "multi",
-          "sort": "desc"
+          "sort": "none"
         }
       },
-      "pluginVersion": "12.1.1",
       "targets": [
         {
           "datasource": {
@@ -1143,14 +1497,15 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "stackdriver_pubsub_subscription_pubsub_googleapis_com_subscription_dead_letter_message_count > 0",
+          "expr": "stackdriver_gcs_bucket_storage_googleapis_com_storage_total_bytes",
           "format": "time_series",
-          "legendFormat": "{{subscription_id}}",
+          "intervalFactor": 2,
+          "legendFormat": "{{bucket_name}} / {{storage_class}}",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Dead Letter Message Count",
+      "title": "GCS Storage Bytes",
       "type": "timeseries"
     }
   ],
@@ -1186,4 +1541,4 @@
   "uid": "stackdriver",
   "version": 3,
   "weekStart": ""
-}
+}

package/src/helm-charts/prom-operator/README-GPU.md

@@ -0,0 +1,42 @@
+# 📌 Optional Prometheus Custom Metrics
+
+## 🎯 Why we do this
+We maintain a **single, consistent Prometheus baseline** for all clusters. Certain metrics - like GPU duty cycle percent or other node-type-specific stats - **do not exist in every cluster**. If we deploy scrape configs or custom adapter rules for optional metrics globally, we risk:
+
+- Empty series queries → scrape flapping.
+- Duplicate or rejected samples.
+- Confusing "phantom target down" or noisy alerts.
+
+## ✅ How we solve it
+- **Baseline:**  
+  Our base `prom-operator` Helm chart and Prometheus Adapter configs **do not contain cluster-specific custom metrics** by default.
+  - running _helmup prom-operator_ will deploy prometheus with the standard exporters
+
+- **Specialized nodes:**  
+  When clusters require optional metrics (e.g., GPU nodes, special sensors, vendor-specific exporters), we deploy those configs with a separate, explicit step:
+  - post helmup run _./prom-operator/hpa-metrics.sh_ to apply the prometheus-adapter.yaml which contains the custom HPA metric rules (GPU duty cycle, Pub/Sub external metrics, ...)
+
+## 🔐 Benefits
+- Zero config drift: our core charts stay generic and versioned.
+- No surprise scrape issues or prometheus TSDB rejects in clusters that don’t expose those metrics.
+- Clusters only carry the scrape configs they can actually fulfill.
+
+## 🛠️ Operational tips
+- **Add custom metrics only when a cluster’s node pool truly provides them.**
+- Keep custom metric configs organized in a clear, dedicated folder (`gpu-adapter-values.yaml`, `gpu-metrics.yaml`, etc.)
+- When adding new clusters, verify whether specialized nodes are present and apply the custom metrics if needed.
+- When GPU adoption grows, we can promote the configs to the baseline and gate them behind a `gpuEnabled` flag.
+
+## ⚙️ Example
+> Our `gpu_duty_cycle_percent` metric is installed *only* on clusters with GPU node pools.  
+> This is done by running:
+>
+> ```bash
+> ./prom-operator/hpa-metrics.sh
+> ```
+> or directly with helm:
+> ```bash
+> helm upgrade prometheus-adapter -f gpu-adapter-values.yaml
+> ```
+>
+> after the base Prometheus stack is installed.

package/src/helm-charts/prom-operator/helmup.plugin

@@ -8,7 +8,7 @@ showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="88.5.4"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="91.4.0"
 helm upgrade $NS --install prometheus-stack $OCI_CHART \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT
@@ -23,7 +23,7 @@ helm upgrade $NS --install elasticsearch8-exporter $OCI_CHART \
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter
 showInstalling "The Stackdriver Exporter (prom-operator GKE metrics exports)"
-[ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="4.13.0"
+[ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="5.2.0"
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/prometheus-stackdriver-exporter"
 helm upgrade $NS --install stackdriver-exporter $OCI_CHART \
                  --values prom-operator/stackdriver-exporter.yaml \