@leverege/build-tools 2.25.2 → 2.26.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.25.2",
+  "version": "2.26.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/helmdn

@@ -87,8 +87,8 @@ GCP_PROJECT_ID=${ADDR[1]} # project id is the second string split by _
 helm repo update &> /dev/null
 
 function annihilateMonitoring() {
-  kubectl delete sa -n monitoring stackdriver-exporter
-  gcloud iam service-accounts delete stackdriver-exporter@${GCP_PROJECT_ID}.iam.gserviceaccount.com --quiet
+  # kubectl delete sa -n monitoring stackdriver-exporter
+  # gcloud iam service-accounts delete stackdriver-exporter@${GCP_PROJECT_ID}.iam.gserviceaccount.com --quiet
 
   # kube-state-metrics?
   helm uninstall -n monitoring grafana prometheus stackdriver-exporter elasticsearch-exporter

package/src/helmup

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-#printf "\n***** RUNNING LOCALLY *****\n" && sleep 2 && alias build-tools="build-tools.js"
+# printf "\n***** RUNNING LOCALLY *****\n" && sleep 2 && alias build-tools="build-tools.js"
 
 # Load the standard helper functions
 . `build-tools --bashfun`
@@ -57,7 +57,7 @@ CATNAMI
   addHelmRepo bitnami $BITNAMI_URL
 }
 
-function bindWorkloadIdentity(){
+function bindWorkloadIdentity() {
   SVC_ACCT=$1
   NAMESPACE=$2
   SVC_ACCT_EMAIL="${SVC_ACCT}@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
@@ -93,12 +93,12 @@ WORKLOAD_ID
   warnOnError $? "Failed to annotate service account $NAMESPACE/$SVC_ACCT - things may fail"
 }
 
-function generatePassword(){
+function generatePassword() {
   RANDOMPW=`openssl rand -base64 12`
   echo $RANDOMPW
 }
 
-function createNamespaceIfNeeded(){
+function createNamespaceIfNeeded() {
   # Check the namespace exists in kubernetes
   kubectl get namespace $1 &> /dev/null
   if [[ $? -eq 1 ]];
@@ -116,7 +116,6 @@ function bootstrapLocalSetup() {
 
   SYSTEM_CHART="$SYSTEM_FILES/$SERVICE-local.yaml"
   [ ! -f "$SYSTEM_CHART" ] && errorExit "Missing $SERVICE chart => $(color y $SYSTEM_CHART)"
-
   SYSTEM_HUP="$SYSTEM_FILES/$SERVICE.hup"
   [ ! -f "$SYSTEM_HUP" ] && errorExit "Missing $SERVICE helmup script => $(color y $SYSTEM_HUP)"
 
@@ -146,8 +145,8 @@ function errorIfMissingSecret() {
   fi
 }
 
-function installMonitoringCharts(){
-  showInstalling "Monitoring Components"
+function installStackdriverExporterEnvironment() {
+  showInstalling "Stackdriver Exporter Service Account Binding"
   createNamespaceIfNeeded monitoring
 
   # Add necessary helm repositories - note that stable is deprecated
@@ -155,9 +154,6 @@ function installMonitoringCharts(){
 
   local SDEXP_SA='stackdriver-exporter'
 
-  # Uninstall the exporter if it is already on the cluster
-  helm uninstall -n monitoring $SDEXP_SA &>/dev/null
-
   # NOTE: The k8s SA != GCP SA - the former is created by the helm chart
   # that installs the stackdriver-exporter, the latter is handled here in
   # order to provide the proper IAM permissions to the exporter.
@@ -174,36 +170,21 @@ function installMonitoringCharts(){
     --role "roles/monitoring.viewer" \
     --member "serviceAccount:$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" &>/dev/null
 
-  printf "\n*** Binding IAM role of workloadIdentityUser to stackdriver-exporter\n"
-  gcloud iam service-accounts add-iam-policy-binding \
-    --project "$GCP_PROJECT_ID" \
-    --role "roles/iam.workloadIdentityUser" \
-    --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[monitoring/$SDEXP_SA]" \
-    "$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
-
-  # The GCP SA is now up but slight delay before launch helm to let the IAM update occur
-  sleep 2
-  helm upgrade --install stackdriver-exporter \
-    prometheus-community/prometheus-stackdriver-exporter \
-    --namespace monitoring \
-    --values stackdriver-exporter/values.yaml \
-    --version 3
+  bindWorkloadIdentity $SDEXP_SA monitoring
 
-  # Verify the Kubernetes SA
-  kubectl get sa -n monitoring $SDEXP_SA
-  if [[ $? -eq 1 ]];
-  then
-    printf "The $SDEXP_SA SA seems to be missing - did startup fail?\n"
-  else
-    # Annotate Kubernetes SA
-    printf "\n*** Annotating the stackdriver-exporter SA\n"
-    kubectl annotate serviceaccount $SDEXP_SA \
-      "iam.gke.io/gcp-service-account=$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
-      --namespace monitoring
-  fi
+  removeHelmRepo prometheus-community
+}
 
-  ## Install Prometheus
-  bootstrapLocalSetup prometheus set-values-local
+function installMonitoringEnvironment() {
+  showInstalling "Monitoring Components"
+  createNamespaceIfNeeded monitoring
+
+  ## Install Stackdriver Exporter
+  installStackdriverExporterEnvironment
+  bootstrapLocalSetup stackdriver-exporter set-values-local
+
+  # Add necessary helm repositories - note that stable is deprecated
+  addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
   ## Installing ES exporter
   helm upgrade --install elasticsearch-exporter prometheus-community/prometheus-elasticsearch-exporter \
@@ -218,6 +199,9 @@ es:
   uri: http://elasticsearch-master.default.svc.cluster.local:9200
 ELASTIC_EXPORTER_CHART_MOD
 
+  ## Install Prometheus
+  bootstrapLocalSetup prometheus set-values-local
+
   ## Install Grafana  
   bootstrapLocalSetup grafana
 
@@ -225,7 +209,7 @@ ELASTIC_EXPORTER_CHART_MOD
   removeHelmRepo prometheus-community
 }
 
-function monitoringHelp(){
+function monitoringHelp() {
   MONPASS=`kubectl get secret --namespace monitoring grafana -o jsonpath='{.data.admin-password}' | base64 --decode`
 
   cat<<ACCESS
@@ -266,9 +250,10 @@ function getManagedSecret() {
 function installPreemptibleKiller() {
   showInstalling "Preemptible Node Killing Coordinator"
   NAMESPACE="estafette"
+  createNamespaceIfNeeded $NAMESPACE
+
   SVC_ACCT="preemptible-killer"
 
-  createNamespaceIfNeeded $NAMESPACE
   addHelmRepo estafette https://helm.estafette.io
 
   printf "\nCreating the gcloud $SVC_ACCT service account (SA)\n"
@@ -691,7 +676,7 @@ function doInstall() {
         ;;
 
       "monitoring")
-        installMonitoringCharts
+        installMonitoringEnvironment
         monitoringHelp
         ;;
 
@@ -718,6 +703,11 @@ function doInstall() {
         removeHelmRepo stakater
         ;;
 
+      "stackdriver-exporter")
+        installStackdriverExporterEnvironment
+        bootstrapLocalSetup stackdriver-exporter set-values-local
+        ;;
+
       "timescale"|"timescale-db")
         bootstrapLocalSetup timescale-db
         ;;

package/src/k8cryo

@@ -1,4 +1,6 @@
 #!/bin/bash
+#
+. `build-tools --bashfun`
 
 if [ "`basename $0`" == "k8thaw" ];
 then

package/src/system-charts/cert-manager.hup

@@ -3,12 +3,12 @@
 # cert-manager may be skipped for now?
 #
 skipInstalling cert-manager && exit 0
-#
+
 addHelmRepo jetstack https://charts.jetstack.io
-[ -z "$CERT_MANAGER_HELM_CHART" ] && CERT_MANAGER_HELM_CHART="v1.10"
+[ -z "$CERT_MANAGER_CHART_VERSION" ] && CERT_MANAGER_CHART_VERSION="v1.10"
 helm upgrade --install cert-manager jetstack/cert-manager \
   --namespace cert-manager --set installCRDs=true \
-  --version $CERT_MANAGER_HELM_CHART
+  --version $CERT_MANAGER_CHART_VERSION
 
 cat<<DELAY
 

package/src/system-charts/grafana.hup

@@ -2,14 +2,12 @@
 #
 showInstalling "Grafana"
 
-createNamespaceIfNeeded monitoring
-
 addHelmRepo grafana https://grafana.github.io/helm-charts
 
-[ -z "$GRAFANA_HELM_CHART" ] && GRAFANA_HELM_CHART="6"
+[ -z "$GRAFANA_CHART_VERSION" ] && GRAFANA_CHART_VERSION="6"
 helm upgrade --install grafana grafana/grafana \
-  --namespace monitoring \
+  --namespace monitoring --create-namespace \
   --values grafana/grafana-local.yaml \
-  --version $GRAFANA_HELM_CHART $HELM_WHAT
+  --version $GRAFANA_CHART_VERSION $HELM_WHAT
 
 removeHelmRepo grafana

package/src/system-charts/prometheus-local.yaml

@@ -1,14 +1,51 @@
+server:
+  global:
+    external_labels:
+      cluster: "OVH:<CLUSTER_NAME>"
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
+  tolerations:
+  - key: "database"
+    operator: "Equal"
+    value: "true"
+    effect: "NoSchedule"  
+  retention: "30d"
+  persistentVolume:
+    size: 50Gi
+  resources:
+    limits:
+      memory: 4Gi
+    requests:
+      memory: 2Gi
+  strategy:
+    type: Recreate
+
+pushgateway:
+  enabled: false
+
+# temp set to false if upgrade errors with something like:
+#   Error: UPGRADE FAILED: cannot patch "prometheus-kube-state-metrics" ... field is immutable
+#
+kubeStateMetrics:
+  enabled: true
+
 alertmanager:
   affinity:
-    nodeAffinity: 
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - preference:
-          matchExpressions:
-          - key: cloud.google.com/gke-nodepool
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
             operator: In
             values:
             - database
-        weight: 100
   tolerations:
   - key: "database"
     operator: "Equal"
@@ -51,6 +88,9 @@ alertmanagerFiles:
     route:
       group_wait: 10s
       group_interval: 5m
+      group_by:
+        - alertname
+        - cluster
       receiver: slack
       routes:
       - match:
@@ -301,38 +341,38 @@ serverFiles:
               description: "{{ $labels.app }} is in a crash loop"
               dashboard: "OVH:<MONITORING>/stackdriver"
 
-      - name: MySQL
-        rules:
-          - alert: MysqlDown
-            expr: mysql_up == 0
-            for: 5m
-            labels:
-              severity: page
-              priority: P3
-            annotations:
-              summary: "MySQL down (instance {{ $labels.instance }})"
-              description: "MySQL pod is down. Look into manually restarting."
-              dashboard: "OVH:<MONITORING>/mysql"
+#     - name: MySQL
+#       rules:
+#         - alert: MysqlDown
+#           expr: mysql_up == 0
+#           for: 5m
+#           labels:
+#             severity: page
+#             priority: P3
+#           annotations:
+#             summary: "MySQL down (instance {{ $labels.instance }})"
+#             description: "MySQL pod is down. Look into manually restarting."
+#             dashboard: "OVH:<MONITORING>/mysql"
 
-          - alert: MysqlSlowQueries
-            expr: mysql_global_status_slow_queries > 0
-            for: 5m
-            labels:
-              severity: warning
-            annotations:
-              summary: "MySQL slow queries detected"
-              description: "MySQL server is detected {{ $value }} slow queries."
-              dashboard: "OVH:<MONITORING>/mysql"
+#         - alert: MysqlSlowQueries
+#           expr: mysql_global_status_slow_queries > 0
+#           for: 5m
+#           labels:
+#             severity: warning
+#           annotations:
+#             summary: "MySQL slow queries detected"
+#             description: "MySQL server is detected {{ $value }} slow queries."
+#             dashboard: "OVH:<MONITORING>/mysql"
 
-          - alert: MysqlRestarted
-            expr: mysql_global_status_uptime < 60
-            for: 5m
-            labels:
-              severity: warning
-            annotations:
-              summary: "MySQL restarted"
-              description: "MySQL has just been restarted, less than one minute ago."
-              dashboard: "OVH:<MONITORING>/mysql"
+#         - alert: MysqlRestarted
+#           expr: mysql_global_status_uptime < 60
+#           for: 5m
+#           labels:
+#             severity: warning
+#           annotations:
+#             summary: "MySQL restarted"
+#             description: "MySQL has just been restarted, less than one minute ago."
+#             dashboard: "OVH:<MONITORING>/mysql"
 
       - name: Postgres
         rules:
@@ -610,15 +650,15 @@ serverFiles:
               summary: "Traefik config failed to reload"
               description: "New config file for Traefik is invalid or Traefik encountered an error"
 
-          - alert: TraefikHighHttp4xxError
-            expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
-            for: 5m
-            labels:
-              severity: error
-            annotations:
-              summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
-              description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
-              dashboard: "OVH:<MONITORING>/traefik"
+#         - alert: TraefikHighHttp4xxError
+#           expr: sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 10
+#           for: 5m
+#           labels:
+#             severity: error
+#           annotations:
+#             summary: "Traefik high HTTP 4xx error rate on {{ $labels.service }}"
+#             description: "Traefik backend 4xx error rate is above 10% (currently at {{ $value }}%)"
+#             dashboard: "OVH:<MONITORING>/traefik"
 
           - alert: TraefikHighHttp5xxError
             expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[5m])) by (service) / sum(rate(traefik_service_requests_total[5m])) by (service) * 100 > 5
@@ -754,39 +794,10 @@ serverFiles:
               summary: "Velero has not made any backups yet"
               description: "No backups have successfully been made since velero was initialized (or restarted), go check it out ASAP"
               dashboard: "OVH:<MONITORING>/velero"
-
-server:
-  affinity:
-    nodeAffinity: 
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - preference:
-          matchExpressions:
-          - key: cloud.google.com/gke-nodepool
-            operator: In
-            values:
-            - database
-        weight: 100
-  tolerations:
-  - key: "database"
-    operator: "Equal"
-    value: "true"
-    effect: "NoSchedule"  
-  retention: "30d"
-  persistentVolume:
-    size: 50Gi
-  resources:
-    limits:
-      memory: 4Gi
-    requests:
-      memory: 2Gi
-  strategy:
-    type: Recreate
-
-pushgateway:
-  enabled: false
-
-# temp set to false if upgrade errors with something like:
-#   Error: UPGRADE FAILED: cannot patch "prometheus-kube-state-metrics" ... field is immutable
 #
-kubeStateMetrics:
-  enabled: true
+# Add local custom rules / alerts here
+#
+#     - name: Custom Local Alerts
+#       rules:
+#         - alert: MyCustomAlertName
+#           expr: my_prom_ql_expression

package/src/system-charts/prometheus.hup

@@ -6,10 +6,10 @@ createNamespaceIfNeeded monitoring
 
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-[ -z "$PROMETHEUS_HELM_CHART" ] && PROMETHEUS_HELM_CHART="15"
+[ -z "$PROMETHEUS_CHART_VERSION" ] && PROMETHEUS_CHART_VERSION="15"
 helm upgrade --install prometheus prometheus-community/prometheus \
              --namespace monitoring \
              --values prometheus/values.yaml \
-             --version $PROMETHEUS_HELM_CHART $HELM_WHAT
+             --version $PROMETHEUS_CHART_VERSION $HELM_WHAT
 
 removeHelmRepo prometheus-community

package/src/system-charts/redis.hup

@@ -4,9 +4,9 @@ showInstalling "Redis"
 
 addBitnamiRepo latest
 
-[ -z "$REDIS_HELM_CHART" ] && REDIS_HELM_CHART="17" # latest chart 17 is redis v7
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="17" # latest chart 17 is redis v7
 helm upgrade --install redis bitnami/redis    \
              --values  redis/redis-local.yaml \
-             --version $REDIS_HELM_CHART $HELM_WHAT
+             --version $REDIS_CHART_VERSION $HELM_WHAT
 
 removeHelmRepo bitnami

package/src/system-charts/stackdriver-exporter-local.yaml

@@ -0,0 +1,8 @@
+stackdriver:
+  projectId: "OVH:<PROJECT_NAME>"
+  metrics:
+    typePrefixes: 'pubsub.googleapis.com/subscription/oldest_unacked_message_age,pubsub.googleapis.com/subscription/num_undelivered_messages,pubsub.googleapis.com/subscription/pull_ack_request_count,pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,pubsub.googleapis.com/topic/message_sizes,pubsub.googleapis.com/topic/send_request_count,firebasedatabase.googleapis.com/io/database_load,firebasedatabase.googleapis.com/network/sent_bytes_count,firebasedatabase.googleapis.com/network/active_connections'
+
+annotations:
+  prometheus.io/port: "9255"
+  prometheus.io/scrape: "true"

package/src/system-charts/stackdriver-exporter.hup

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+showInstalling "Stackdriver Exporter"
+
+addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
+
+[ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="4"
+helm upgrade --install stackdriver-exporter \
+  prometheus-community/prometheus-stackdriver-exporter \
+  --namespace monitoring --create-namespace \
+  --values stackdriver-exporter/values.yaml \
+  --version $STACKDRIVER_EXPORTER_CHART_VERSION $HELM_WHAT
+
+removeHelmRepo prometheus-community

package/src/system-charts/timescale-db.hup

@@ -4,7 +4,7 @@ showInstalling "Timescale Database (Leverege chart)"
 
 errorIfMissingSecret postgresql-password
 
-[ -z "$TIMESCALE_HELM_CHART" ] && TIMESCALE_HELM_CHART="2"
+[ -z "$TIMESCALE_DB_CHART_VERSION" ] && TIMESCALE_DB_CHART_VERSION="2"
 helm upgrade --install timescale-db leverege/timescale-db   \
              --values  timescale-db/timescale-db-local.yaml \
-             --version $TIMESCALE_HELM_CHART $HELM_WHAT
+             --version $TIMESCALE_DB_CHART_VERSION $HELM_WHAT

package/src/system-charts/traefik-canary.hup

@@ -4,13 +4,12 @@ showInstalling "Traefik Canary Load Balancer / Router"
 
 TRAEFIK_CANARY="traefik-canary"
 
-createNamespaceIfNeeded $TRAEFIK_CANARY
-
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_HELM_CHART" ] && TRAEFIK_HELM_CHART="16"
-helm upgrade --install $TRAEFIK_CANARY traefik/traefik --namespace $TRAEFIK_CANARY \
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="16"
+helm upgrade --install $TRAEFIK_CANARY traefik/traefik \
+             --namespace $TRAEFIK_CANARY --create-namespace \
              --values  $TRAEFIK_CANARY/traefik-canary-local.yaml \
-             --version $TRAEFIK_HELM_CHART $HELM_WHAT
+             --version $TRAEFIK_CHART_VERSION $HELM_WHAT
 
 removeHelmRepo traefik

package/src/system-charts/traefik.hup

@@ -3,14 +3,13 @@
 showInstalling "Traefik Load Balancer / Router"
 
 TRAEFIK_NAMESPACE="traefik"
-createNamespaceIfNeeded $TRAEFIK_NAMESPACE
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_HELM_CHART" ] && TRAEFIK_HELM_CHART="16"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="16"
 helm upgrade --install traefik traefik/traefik \
-             --namespace $TRAEFIK_NAMESPACE \
+             --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/traefik-local.yaml \
-             --version $TRAEFIK_HELM_CHART $HELM_WHAT
+             --version $TRAEFIK_CHART_VERSION $HELM_WHAT
 
 removeHelmRepo traefik

package/src/system-charts/velero.hup

@@ -1,8 +1,10 @@
 #!/bin/bash
+#
+showInstalling "Velero Backup System"
 
 addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 
-[ -z "$VELERO_HELM_CHART" ] && VELERO_HELM_CHART="3"
+[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="3"
 #
 # Build the bucket, region and SA email variables and use --set to mod the
 # chart values as opposed to using an OVH:<label> approach. This eliminates
@@ -11,7 +13,7 @@ addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 #
 # Note: GCP_PROJECT_ID is passed down from helmup's environment.
 #
-GCE_BUCKET="$GCP_PROJECT_ID-velero-$VELERO_HELM_CHART"
+GCE_BUCKET="$GCP_PROJECT_ID-velero-$VELERO_CHART_VERSION"
 GCE_REGION=`overwhelm -k GCE_REGION`
 GCE_SA_EMAIL="velero@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
 
@@ -21,9 +23,10 @@ GCE_SA_EMAIL="velero@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
 velero schedule delete --all --confirm &> /dev/null
 
 helm upgrade \
-     --install velero vmware-tanzu/velero --namespace velero \
+     --install velero vmware-tanzu/velero \
+     --namespace velero --create-namespace \
      --values velero/velero-local.yaml \
-     --version $VELERO_HELM_CHART $HELM_WHAT \
+     --version $VELERO_CHART_VERSION $HELM_WHAT \
      --set configuration.backupStorageLocation.bucket="$GCE_BUCKET" \
      --set configuration.backupStorageLocation.config.serviceAccount="$GCE_SA_EMAIL" \
      --set configuration.volumeSnapshotLocation.config.snapshotLocation="$GCE_REGION"
@@ -35,6 +38,7 @@ cat<<DONE_VELERO
 
   Wait at least 1 minute before kicking off the following backups:
 
+  `color g 'velero backup create --from-schedule velero-full'`
   `color g 'velero backup create --from-schedule velero-psql'`
   `color g 'velero backup create --from-schedule velero-tsdb'`