@leverege/build-tools 2.14.1 → 2.15.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.14.1",
+  "version": "2.15.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -61,21 +61,21 @@
     "inquirer": "^8.2.4",
     "js-yaml": "^4.1.0",
     "ncp": "^2.0.0",
-    "npm-check-updates": "^13.1.1",
+    "npm-check-updates": "^13.1.2",
     "npm-registry-fetch": "^11.0.0",
     "open": "^8.4.0",
     "ora": "^5.4.1",
     "parse-gitignore": "^1.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.3.7",
-    "simple-git": "^3.7.1"
+    "simple-git": "^3.9.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.17.10",
-    "@babel/core": "^7.18.2",
-    "@babel/node": "^7.17.10",
+    "@babel/core": "^7.18.5",
+    "@babel/node": "^7.18.5",
     "@leverege/babel-preset-leverege-node": "^2.0.0",
     "@leverege/eslint-config-leverege": "^3.0.1",
-    "npm": "^8.12.1"
+    "npm": "^8.12.2"
   }
 }

package/src/bash-funcs

@@ -95,6 +95,8 @@ INSTALLING
 }
 
 function addHelmRepo() {
+  [ ! -z "$HELM_REPO_SKIP" ] && return
+
   local chart=$1
   local url=$2
 
@@ -107,5 +109,6 @@ REPOADD
 }
 
 function removeHelmRepo(){
+  [ ! -z "$HELM_REPO_SKIP" ] && return
   helm repo remove $1 &>/dev/null
 }

package/src/helmdn

@@ -52,6 +52,7 @@ PLATFORM=(
   scheduler
   transponder-bq
   transponder-postgres
+  transponder-tsdb
   transponder-rt
 )
 

package/src/helmup

@@ -8,7 +8,7 @@
 # Load the standard helper functions
 . `build-tools --bashfun`
 
-HELMUP_ARGS=("$*")
+SHELMUP_ARGS=("$*")
 
 # removed geotile-server from standard platform 3/20/22 - low usage
 PLATFORM=(
@@ -25,6 +25,7 @@ PLATFORM=(
   transponder-bq
   transponder-rt
   transponder-postgres
+  transponder-tsdb
 )
 
 SYSTEM=(
@@ -59,6 +60,52 @@ CATNAMI
   addHelmRepo bitnami $BITNAMI_URL
 }
 
+function getProjectIdFromK8sContext() {
+  local currContext=`kubectl config current-context`
+   
+  IFS='_'
+  read -ra ADDR <<< "$currContext" 
+
+  echo ${ADDR[1]} # project id is the second string split by _
+}
+
+function bindWorkloadIdentity(){
+  PROJECT_ID=$(getProjectIdFromK8sContext)
+  SVC_ACCT=$1
+  NAMESPACE=$2
+  SVC_ACCT_EMAIL="${SVC_ACCT}@${PROJECT_ID}.iam.gserviceaccount.com"
+
+  cat<<WORKLOAD_ID
+Binding workload identity IAM policy
+  Service Account : `color g $SVC_ACCT`
+  Project         : `color g $PROJECT_ID`
+  Namespace       : `color g $NAMESPACE`
+  Service Email   : `color g $SVC_ACCT_EMAIL`
+
+WORKLOAD_ID
+
+  gcloud iam service-accounts add-iam-policy-binding \
+      --project $PROJECT_ID \
+      --member "serviceAccount:$PROJECT_ID.svc.id.goog[${NAMESPACE}/${SVC_ACCT}]" \
+      --role roles/iam.workloadIdentityUser $SVC_ACCT_EMAIL
+  warnOnError $? "gcloud workload identity binding may have failed"
+ 
+  # Verify the Kubernetes SA and create it if necessary
+  printf "\nChecking for the $NAMESPACE/#SVC_ACCT k8s SA\n"
+  kubectl get sa -n $NAMESPACE $SVC_ACCT
+  if [[ $? -eq 1 ]];
+  then
+    printf "Creating the k8s SA $NAMESPACE/@$SVC_ACCT\n"
+    kubectl create sa -n $NAMESPACE $SVC_ACCT
+  fi
+
+  printf "\nAnnotating the k8s SA $NAMESPACE/@$SVC_ACCT\n"
+  kubectl annotate serviceaccount $SVC_ACCT \
+      --namespace $NAMESPACE --overwrite \
+      "iam.gke.io/gcp-service-account=$SVC_ACCT_EMAIL"
+  warnOnError $? "Failed to annotate service account $NAMESPACE/$SVC_ACCT - things may fail"
+}
+
 function checkTimescalePassword(){
   kubectl get secret postgresql-password
   if [[ $? -eq 1 ]]; then
@@ -90,8 +137,20 @@ function createNamespaceIfNeeded(){
   fi
 }
 
+function installPrometheus() {
+  # Add the community helm repositories
+  addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
+
+  helm upgrade --install prometheus prometheus-community/prometheus \
+    --namespace monitoring \
+    --values prometheus/values.yaml \
+    --version 13
+
+# removeHelmRepo prometheus-community
+}
+
 function installMonitoringCharts(){
-  showInstalling "Grafana Monitoring"
+  showInstalling "Monitoring Components"
   createNamespaceIfNeeded monitoring
 
   # Add necessary helm repositories - note that stable is deprecated
@@ -102,8 +161,8 @@ function installMonitoringCharts(){
    
   IFS='_'
   read -ra ADDR <<< "$currContext" 
-
   KUBERNETES_ENV=${ADDR[0]} # should be either gke or eks
+  GCP_PROJECT_ID=${ADDR[1]} # project id is the second string split by _
 
   ## Installing stackdriver exporter in GKE
   if [[ "$KUBERNETES_ENV" = "gke" ]];
@@ -115,28 +174,16 @@ function installMonitoringCharts(){
 
   The stackdriver-exporter SA already exists which means installing the
   helm chart is going to fail because it wants to manage that aspect of the
-  system.
+  system. This most likely because the stackdriver exporter is already
+  installed and running. Run the following helm command to uninstall first:
+
+      `color y "overwhelm && helm uninstall -n monitoring stackdriver-exporter"`
 
 SA_EXISTS
-      kubectl get sa -n monitoring stackdriver-exporter
-      exitOnError 1 "Must remove the SA in order to proceed - `color r DANGER`"
+      exitOnError 1 "uninstall the existing deployment in order to proceed"
     fi
 
-    GCP_PROJECT_ID=${ADDR[1]} # project id is the second string split by _
     SDEXP_SA='stackdriver-exporter'
-    helm upgrade --install stackdriver-exporter \
-      prometheus-community/prometheus-stackdriver-exporter \
-      --namespace monitoring \
-      --values stackdriver-exporter/values.yaml \
-      --version 1.12.0
-
-    # Verify the Kubernetes SA
-    kubectl get sa -n monitoring $SDEXP_SA
-    if [[ $? -eq 1 ]];
-    then
-      printf "The $SDEXP_SA SA seems to be missing, which is weird - creating it now\n"
-      kubectl create sa -n monitoring stackdriver-exporter
-    fi
 
     # NOTE: The k8s SA != GCP SA - the former is created by the helm chart
     # that installs the stackdriver-exporter, the latter is handled here in
@@ -145,7 +192,8 @@ SA_EXISTS
     # Create GCP SA and bind policies
     printf "\n*** Creating GCP SA with monitoring/viewer ***\n"
     gcloud iam service-accounts create $SDEXP_SA \
-      --project "$GCP_PROJECT_ID"
+      --project "$GCP_PROJECT_ID" \
+      --display-name "Prometheus Stackdriver Exporter"
     warnOnError $? "the GCP SA $SDEXP_SA may already exist"
 
     printf "\n*** Binding IAM role of monitoring viewer to stackdriver-exporter in $GCP_PROJECT_ID\n"
@@ -160,31 +208,43 @@ SA_EXISTS
       --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[monitoring/$SDEXP_SA]" \
       "$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
 
-    # Annotate Kubernetes SA
-    printf "\n*** Annotating the stackdriver-exporter SA\n"
-    kubectl annotate serviceaccount $SDEXP_SA \
-      "iam.gke.io/gcp-service-account=$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
-      --namespace monitoring
+    # The GCP SA is now up but slight delay before launch helm to let the IAM update occur
+    sleep 2
+    helm upgrade --install stackdriver-exporter \
+      prometheus-community/prometheus-stackdriver-exporter \
+      --namespace monitoring \
+      --values stackdriver-exporter/values.yaml \
+      --version 3
+
+    # Verify the Kubernetes SA
+    kubectl get sa -n monitoring $SDEXP_SA
+    if [[ $? -eq 1 ]];
+    then
+      printf "The $SDEXP_SA SA seems to be missing - did startup fail?\n"
+    else
+      # Annotate Kubernetes SA
+      printf "\n*** Annotating the stackdriver-exporter SA\n"
+      kubectl annotate serviceaccount $SDEXP_SA \
+        "iam.gke.io/gcp-service-account=$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
+        --namespace monitoring
+    fi
   fi
 
   ## Install Prometheus
-  helm upgrade --install prometheus prometheus-community/prometheus \
-    --namespace monitoring \
-    --values prometheus/values.yaml \
-    --version 13
+  installPrometheus
 
   ## Installing ES exporter
   helm upgrade --install elasticsearch-exporter prometheus-community/prometheus-elasticsearch-exporter \
     --namespace monitoring \
     --values elasticsearch-exporter/values-es-exporter.yaml \
-    --version 4.11
+    --version 4.13
 
   ## Install Grafana  
   addHelmRepo grafana https://grafana.github.io/helm-charts
   helm upgrade --install grafana grafana/grafana \
     --namespace monitoring \
     --values grafana/values-grafana.yaml \
-    --version 6.26.5 # latest v8 grafana
+    --version 6 # latest v8 grafana
 
   ## Clean up
   removeHelmRepo prometheus-community
@@ -253,45 +313,42 @@ function getManagedSecret() {
   echo $SECRET
 }
 
-function getProjectIdFromK8sContext() {
-  local currContext=`kubectl config current-context`
-   
-  IFS='_'
-  read -ra ADDR <<< "$currContext" 
-
-  echo ${ADDR[1]} # project id is the second string split by _
-}
-
 # https://github.com/estafette/estafette-gke-preemptible-killer
 function installPreemptibleKiller() {
   showInstalling "Preemptible Node Killing Adjusticator"
+  NAMESPACE="estafette"
   PROJECT_ID="$(getProjectIdFromK8sContext)"
-  createNamespaceIfNeeded estafette
+  SVC_ACCT="preemptible-killer"
+
+  createNamespaceIfNeeded $NAMESPACE
   addHelmRepo estafette https://helm.estafette.io
 
-  gcloud iam service-accounts create preemptible-killer \
-    --project=$PROJECT_ID \
-    --display-name preemptible-killer
+  printf "\nCreating the gcloud $SVC_ACCT service account (SA)\n"
+  gcloud iam service-accounts create $SVC_ACCT \
+    --project $PROJECT_ID \
+  warnOnError $? "SA creation may have failed or it already exists"
 
-  gcloud iam roles create preemptible_killer \
+  printf "\nCreate the $SVC_ACCT IAM role\n"
+  gcloud iam roles create preemptibleKiller \
     --project $PROJECT_ID \
-    --title preemptible-killer \
+    --title "Estafette Node Killer" \
     --description "Delete compute instances" \
     --permissions compute.instances.delete
+  warnOnError $? "Role creation may have failed or it already exists"
 
-  local service_account_email=$(gcloud iam --project=$PROJECT_ID service-accounts list --filter preemptible-killer --format 'value([email])')
+  local service_account_email=$(gcloud iam --project=$PROJECT_ID service-accounts list --filter $SVC_ACCT --format 'value([email])')
+  printf "\nBinding the IAM role to the $SVC_ACCT SA\n"
   gcloud projects add-iam-policy-binding $PROJECT_ID \
     --member=serviceAccount:${service_account_email} \
-    --role=projects/${PROJECT_ID}/roles/preemptible_killer &>/dev/null
+    --role=projects/${PROJECT_ID}/roles/preemptibleKiller &> /dev/null
+  warnOnError $? "Binding may have failed or it already exists"
 
-  gcloud iam service-accounts keys create \
-    --project=$PROJECT_ID \
-    --iam-account $service_account_email \
-    google_service_account.json &>/dev/null
+  bindWorkloadIdentity $SVC_ACCT $NAMESPACE
 
   helm upgrade --install estafette-gke-preemptible-killer \
                estafette/estafette-gke-preemptible-killer \
-               --namespace estafette \
+               --namespace ${NAMESPACE} \
+               --set secret.workloadIdentityServiceAccount="$SVC_ACCT@$PROJECT_ID.iam.gserviceaccount.com" \
                --set extraEnv.BLACKLIST_HOURS="10:00-02:00" # No killing from 5AM-9PM EST
 #              --set extraEnv.WHITELIST_HOURS="12:30-13:30"
   [ -f "google_service_account.json" ] && rm google_service_account.json
@@ -403,7 +460,7 @@ NEED_MANAGED_SECRET
   addHelmRepo jetstack https://charts.jetstack.io
   helm upgrade --install cert-manager jetstack/cert-manager \
     --namespace cert-manager --set installCRDs=true \
-    --version v1.7.0
+    --version v1.8
 
   cat<<DELAY
 
@@ -478,10 +535,12 @@ TRAEFIK_CFG
 
   addHelmRepo traefik https://helm.traefik.io/traefik
 
+  [ -z "$TRAEFIK_HELM_CHART" ] && TRAEFIK_HELM_CHART="10"
   helm upgrade --install traefik traefik/traefik \
     --namespace traefik \
-    --values $TRAEFIK_HA_YAML
-
+    --version  $TRAEFIK_HELM_CHART \
+    --values   $TRAEFIK_HA_YAML
+ 
   removeHelmRepo traefik
 }
 
@@ -579,7 +638,7 @@ function installVelero() {
   
   kubectl annotate serviceaccount velero \
     --namespace velero \
-    iam.gke.io/gcp-service-account=$SERVICE_ACCOUNT_EMAIL
+    "iam.gke.io/gcp-service-account=$SERVICE_ACCOUNT_EMAIL"
   warnOnError $? "Failed to annotate k8s SA - things may fail"
 
   gsutil iam ch serviceAccount:$SERVICE_ACCOUNT_EMAIL:objectAdmin gs://$BUCKET
@@ -591,6 +650,7 @@ function installVelero() {
   ## Helpful debugging link => https://github.com/vmware-tanzu/helm-charts/issues/351
   addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
   helm upgrade --install velero vmware-tanzu/velero \
+    --version 2.29 \
     --namespace velero \
     --set credentials.useSecret="false" \
     --set configuration.provider="gcp" \
@@ -745,6 +805,7 @@ function doInstall() {
         checkSSDStorageClass
         addHelmRepo elastic https://helm.elastic.co
         helm upgrade --install elasticsearch elastic/elasticsearch \
+          --version 7 \
           --values $LOCAL_ES_VALUES \
           --set volumeClaimTemplate.storageClassName="ssd"
         removeHelmRepo elastic
@@ -770,12 +831,17 @@ function doInstall() {
         removeHelmRepo bitnami
         ;;
 
+      "prometheus")
+        installPrometheus
+        ;;
+
       "redis")
         showInstalling "Redis Network Cache"
         addBitnamiRepo
         [ -z "$REDIS_HELM_CHART" ] && REDIS_HELM_CHART="13.0.1" # latest breaks us => "v14+"
         helm upgrade --install redis bitnami/redis \
           -f redis/redis-values.yaml \
+          --set auth.enabled=false \
           --version $REDIS_HELM_CHART
         printf "\n\nTo start a specific version => `color g 'REDIS_HELM_CHART="13.0.1" helmup redis'`\n\n"
         removeHelmRepo bitnami