@leverege/build-tools 2.22.0 → 2.23.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.22.0",
+  "version": "2.23.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -67,8 +67,8 @@
     "zx": "^7.1.1"
   },
   "devDependencies": {
-    "@babel/cli": "^7.19.3",
-    "@babel/core": "^7.20.5",
+    "@babel/cli": "^7.20.7",
+    "@babel/core": "^7.20.12",
     "@leverege/babel-preset-leverege-node": "^2.0.0",
     "@leverege/eslint-config-leverege": "^3.0.1",
     "npm": "^9.2.0"

package/src/bash-funcs

@@ -84,6 +84,15 @@ function getManagedSecret() {
   echo $SECRET
 }
 
+function getProjectIdFromK8sContext() {
+  local currContext=`kubectl config current-context`
+
+  IFS='_'
+  read -ra ADDR <<< "$currContext"
+
+  echo ${ADDR[1]} # project id is the second string split by _
+}
+
 function showInstalling() {
   cat<<INSTALLING
 
@@ -94,6 +103,16 @@ $(color y '=====================================================================
 INSTALLING
 }
 
+function skipInstalling() {
+  cat<<SKIPPING
+
+$(color r '======================================================================')
+
+  Skipping => $(color y "$1")
+
+SKIPPING
+}
+
 function addHelmRepo() {
   local chart=$1
   local url=$2
@@ -102,7 +121,7 @@ function addHelmRepo() {
   Adding helm repo => $(color g "$chart") from $(color g "$url")
 
 REPOADD
-  helm repo add $chart $url &> /dev/null
+  helm repo add --force-update $chart $url &> /dev/null
   helm repo update &> /dev/null
 }
 

package/src/helmup

@@ -7,6 +7,9 @@
 
 SHELMUP_ARGS=("$*")
 
+# The project id is used in a lot of places so just set it here
+GCP_PROJECT_ID="$(getProjectIdFromK8sContext)"
+
 # removed geotile-server from standard platform 3/20/22 - low usage
 PLATFORM=(
   authz-server
@@ -21,7 +24,7 @@ PLATFORM=(
   scheduler
   transponder-bq
   transponder-rt
-  transponder-postgres
+  transponder-postgres # deprecated
   transponder-tsdb
 )
 
@@ -33,7 +36,7 @@ SYSTEM=(
   preemptible-killer
   zombie-killer
   reloader
-  cert-manager
+  cert-manager # deprecated
   traefik
   velero
   monitoring # do monitoring last to show password
@@ -57,33 +60,23 @@ CATNAMI
   addHelmRepo bitnami $BITNAMI_URL
 }
 
-function getProjectIdFromK8sContext() {
-  local currContext=`kubectl config current-context`
-   
-  IFS='_'
-  read -ra ADDR <<< "$currContext" 
-
-  echo ${ADDR[1]} # project id is the second string split by _
-}
-
 function bindWorkloadIdentity(){
-  PROJECT_ID=$(getProjectIdFromK8sContext)
   SVC_ACCT=$1
   NAMESPACE=$2
-  SVC_ACCT_EMAIL="${SVC_ACCT}@${PROJECT_ID}.iam.gserviceaccount.com"
+  SVC_ACCT_EMAIL="${SVC_ACCT}@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
 
   cat<<WORKLOAD_ID
 Binding workload identity IAM policy
   Service Account : `color g $SVC_ACCT`
-  Project         : `color g $PROJECT_ID`
+  Project         : `color g $GCP_PROJECT_ID`
   Namespace       : `color g $NAMESPACE`
   Service Email   : `color g $SVC_ACCT_EMAIL`
 
 WORKLOAD_ID
 
   gcloud iam service-accounts add-iam-policy-binding \
-      --project $PROJECT_ID \
-      --member "serviceAccount:$PROJECT_ID.svc.id.goog[${NAMESPACE}/${SVC_ACCT}]" \
+      --project $GCP_PROJECT_ID \
+      --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[${NAMESPACE}/${SVC_ACCT}]" \
       --role roles/iam.workloadIdentityUser $SVC_ACCT_EMAIL
   warnOnError $? "gcloud workload identity binding may have failed"
  
@@ -111,7 +104,8 @@ function generatePassword(){
 function createNamespaceIfNeeded(){
   # Check the namespace exists in kubernetes
   kubectl get namespace $1 &> /dev/null
-  if [[ $? -eq 1 ]]; then
+  if [[ $? -eq 1 ]];
+  then
     printf "\n*** Creating $1 namespace ***\n\n"
     kubectl create namespace $1
   fi
@@ -157,20 +151,12 @@ function installGrafana() {
   removeHelmRepo grafana
 }
 
-function checkTimescalePassword() {
-  kubectl get secret postgresql-password
-  if [[ $? -eq 1 ]]; then
-    printf "\n*** Autogenerating Timescale password ***\n\n"
-    local psql_pw="$(generatePassword)"
-    local psql_rep_pw="$(generatePassword)"
-
-    kubectl create secret generic postgresql-password \
-      --from-literal=postgresql-password=${psql_pw} \
-      --from-literal=postgresql-replication-password=${psql_rep_pw}
-
-    printf "\n\n`color y '*** Save Timescale creds to Keeper ***'`\n\n"
-    printf "`color g "postgresql-password / ${psql_pw}"`\n"
-    printf "`color g "postgresql-replication-password / ${psql_rep_pw}"`\n\n"
+function errorIfMissingSecret() {
+  SECRET="$1"
+  kubectl get secret $SECRET &> /dev/null
+  if [[ $? = 1 || SECRET = "" ]];
+  then
+    errorExit "missing secret `color y $SECRET` - did you `color g 'service-man --new-k8s'` ?"
   fi
 }
 
@@ -181,82 +167,57 @@ function installMonitoringCharts(){
   # Add necessary helm repositories - note that stable is deprecated
   addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-  ## Set the project id by parsing current context
-  currContext=`kubectl config current-context`
-   
-  IFS='_'
-  read -ra ADDR <<< "$currContext" 
-  KUBERNETES_ENV=${ADDR[0]} # should be either gke or eks
-  GCP_PROJECT_ID=${ADDR[1]} # project id is the second string split by _
+  local SDEXP_SA='stackdriver-exporter'
 
-  ## Installing stackdriver exporter in GKE
-  if [[ "$KUBERNETES_ENV" = "gke" ]];
-  then
-    kubectl get sa -n monitoring stackdriver-exporter &> /dev/null
-    if [[ $? -eq 0 ]];
-    then
-      cat<<SA_EXISTS
-
-  The stackdriver-exporter SA already exists which means installing the
-  helm chart is going to fail because it wants to manage that aspect of the
-  system. This most likely because the stackdriver exporter is already
-  installed and running. Run the following helm command to uninstall first:
+  # Uninstall the exporter if it is already on the cluster
+  helm uninstall -n monitoring $SDEXP_SA &>/dev/null
 
-      `color y "overwhelm && helm uninstall -n monitoring stackdriver-exporter"`
-
-SA_EXISTS
-      exitOnError 1 "uninstall the existing deployment in order to proceed"
-    fi
+  # NOTE: The k8s SA != GCP SA - the former is created by the helm chart
+  # that installs the stackdriver-exporter, the latter is handled here in
+  # order to provide the proper IAM permissions to the exporter.
 
-    SDEXP_SA='stackdriver-exporter'
+  # Create GCP SA and bind policies
+  printf "\n*** Creating GCP SA with monitoring/viewer ***\n"
+  gcloud iam service-accounts create $SDEXP_SA \
+    --project "$GCP_PROJECT_ID" \
+    --display-name "Prometheus Stackdriver Exporter" &>/dev/null
+  warnOnError $? "the GCP SA $SDEXP_SA may already exist"
 
-    # NOTE: The k8s SA != GCP SA - the former is created by the helm chart
-    # that installs the stackdriver-exporter, the latter is handled here in
-    # order to provide the proper IAM permissions to the exporter.
+  printf "\n*** Binding IAM role of monitoring viewer to stackdriver-exporter in $GCP_PROJECT_ID\n"
+  gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \
+    --role "roles/monitoring.viewer" \
+    --member "serviceAccount:$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" &>/dev/null
 
-    # Create GCP SA and bind policies
-    printf "\n*** Creating GCP SA with monitoring/viewer ***\n"
-    gcloud iam service-accounts create $SDEXP_SA \
-      --project "$GCP_PROJECT_ID" \
-      --display-name "Prometheus Stackdriver Exporter"
-    warnOnError $? "the GCP SA $SDEXP_SA may already exist"
-
-    printf "\n*** Binding IAM role of monitoring viewer to stackdriver-exporter in $GCP_PROJECT_ID\n"
-    gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \
-      --role "roles/monitoring.viewer" \
-      --member "serviceAccount:$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" &>/dev/null
-
-    printf "\n*** Binding IAM role of workloadIdentityUser to stackdriver-exporter\n"
-    gcloud iam service-accounts add-iam-policy-binding \
-      --project "$GCP_PROJECT_ID" \
-      --role "roles/iam.workloadIdentityUser" \
-      --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[monitoring/$SDEXP_SA]" \
-      "$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
+  printf "\n*** Binding IAM role of workloadIdentityUser to stackdriver-exporter\n"
+  gcloud iam service-accounts add-iam-policy-binding \
+    --project "$GCP_PROJECT_ID" \
+    --role "roles/iam.workloadIdentityUser" \
+    --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[monitoring/$SDEXP_SA]" \
+    "$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
+
+  # The GCP SA is now up but slight delay before launch helm to let the IAM update occur
+  sleep 2
+  helm upgrade --install stackdriver-exporter \
+    prometheus-community/prometheus-stackdriver-exporter \
+    --namespace monitoring \
+    --values stackdriver-exporter/values.yaml \
+    --version 3
 
-    # The GCP SA is now up but slight delay before launch helm to let the IAM update occur
-    sleep 2
-    helm upgrade --install stackdriver-exporter \
-      prometheus-community/prometheus-stackdriver-exporter \
-      --namespace monitoring \
-      --values stackdriver-exporter/values.yaml \
-      --version 3
-
-    # Verify the Kubernetes SA
-    kubectl get sa -n monitoring $SDEXP_SA
-    if [[ $? -eq 1 ]];
-    then
-      printf "The $SDEXP_SA SA seems to be missing - did startup fail?\n"
-    else
-      # Annotate Kubernetes SA
-      printf "\n*** Annotating the stackdriver-exporter SA\n"
-      kubectl annotate serviceaccount $SDEXP_SA \
-        "iam.gke.io/gcp-service-account=$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
-        --namespace monitoring
-    fi
+  # Verify the Kubernetes SA
+  kubectl get sa -n monitoring $SDEXP_SA
+  if [[ $? -eq 1 ]];
+  then
+    printf "The $SDEXP_SA SA seems to be missing - did startup fail?\n"
+  else
+    # Annotate Kubernetes SA
+    printf "\n*** Annotating the stackdriver-exporter SA\n"
+    kubectl annotate serviceaccount $SDEXP_SA \
+      "iam.gke.io/gcp-service-account=$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
+      --namespace monitoring
   fi
 
   ## Install Prometheus
-  installPrometheus
+  bootstrapLocalSetup prometheus set-values-local
 
   ## Installing ES exporter
   helm upgrade --install elasticsearch-exporter prometheus-community/prometheus-elasticsearch-exporter \
@@ -272,7 +233,7 @@ es:
 ELASTIC_EXPORTER_CHART_MOD
 
   ## Install Grafana  
-  installGrafana
+  bootstrapLocalSetup grafana
 
   ## Clean up
   removeHelmRepo prometheus-community
@@ -295,38 +256,13 @@ function monitoringHelp(){
     kubectl --namespace monitoring port-forward \$GRAFANA_POD 3000 &
     sleep 2; open http://localhost:3000
 
-  `color toast "Save monitoring login to Keeweb"`   => `color r "admin / $MONPASS"`
+  `color toast "Save monitoring login to Keeper"`   => `color r "admin / $MONPASS"`
 
   `color g "Show this helpful message again with"` => `color r "helmup monitoring-wtf"`
 
 ACCESS
 }
 
-function createSSDYAML(){
-  cat<<EOSSD > ${K8S_UTILS}/ssd-storageclass.yaml
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: ssd
-provisioner: kubernetes.io/gce-pd
-parameters:
-  type: pd-ssd
-EOSSD
-}
-
-function checkSSDStorageClass(){
-  kubectl get storageclass ssd
-  if [[ $? -eq 1 ]]; then
-    printf "\n*** SSD Storage Class not defined ***\n\n"
-    if [ ! -f ${K8S_UTILS}/ssd-storageclass.yaml ]; then
-      createSSDYAML
-    fi
-    kubectl apply -f ${K8S_UTILS}/ssd-storageclass.yaml
-  fi
-
-  printf "\n*** Attaching PVC to SSD Class ***\n\n"
-}
-
 function getManagedSecret() {
   local SECNAME=$1
   local PROJECT="${2:-leverege-docker-images}"
@@ -344,7 +280,6 @@ function getManagedSecret() {
 function installPreemptibleKiller() {
   showInstalling "Preemptible Node Killing Coordinator"
   NAMESPACE="estafette"
-  PROJECT_ID="$(getProjectIdFromK8sContext)"
   SVC_ACCT="preemptible-killer"
 
   createNamespaceIfNeeded $NAMESPACE
@@ -352,24 +287,24 @@ function installPreemptibleKiller() {
 
   printf "\nCreating the gcloud $SVC_ACCT service account (SA)\n"
   gcloud iam service-accounts create $SVC_ACCT \
-    --project $PROJECT_ID \
+    --project $GCP_PROJECT_ID \
     --description "Estafette Node Killer SA" \
     --display-name "Estafette Node Killer SA"
   warnOnError $? "SA creation may have failed or it already exists"
 
   printf "\nCreate the $SVC_ACCT IAM role\n"
   gcloud iam roles create preemptibleKiller \
-    --project $PROJECT_ID \
+    --project $GCP_PROJECT_ID \
     --title "Estafette Node Killer" \
     --description "Delete compute instances" \
     --permissions compute.instances.delete
   warnOnError $? "Role creation may have failed or it already exists"
 
-  local service_account_email=$(gcloud iam --project=$PROJECT_ID service-accounts list --filter $SVC_ACCT --format 'value([email])')
+  local service_account_email=$(gcloud iam --project=$GCP_PROJECT_ID service-accounts list --filter $SVC_ACCT --format 'value([email])')
   printf "\nBinding the IAM role to the $SVC_ACCT SA\n"
-  gcloud projects add-iam-policy-binding $PROJECT_ID \
+  gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
     --member=serviceAccount:${service_account_email} \
-    --role=projects/${PROJECT_ID}/roles/preemptibleKiller &> /dev/null
+    --role=projects/${GCP_PROJECT_ID}/roles/preemptibleKiller &> /dev/null
   warnOnError $? "Binding may have failed or it already exists"
 
   bindWorkloadIdentity $SVC_ACCT $NAMESPACE
@@ -377,7 +312,7 @@ function installPreemptibleKiller() {
   helm upgrade --install estafette-gke-preemptible-killer \
                estafette/estafette-gke-preemptible-killer \
                --namespace ${NAMESPACE} \
-               --set secret.workloadIdentityServiceAccount="$SVC_ACCT@$PROJECT_ID.iam.gserviceaccount.com" \
+               --set secret.workloadIdentityServiceAccount="$SVC_ACCT@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
                --set extraEnv.BLACKLIST_HOURS="10:00-02:00" # No killing from 5AM-9PM EST
 #              --set extraEnv.WHITELIST_HOURS="12:30-13:30"
   [ -f "google_service_account.json" ] && rm google_service_account.json
@@ -466,14 +401,15 @@ function installCertManager() {
   createNamespaceIfNeeded cert-manager
 
   kubectl get secret cloudflare --namespace cert-manager &>/dev/null
-  if [[ $? -eq 1 ]]; then
+  if [[ $? -eq 1 ]];
+  then
     cat<<NEED_MANAGED_SECRET
    $RED_ERROR missing secret: `color y cloudflare` namespace: `color y cert-manager`
 
-   This secret may be fetched from the leverege-docker-images cluster and
-   properly injected into this cluster.
+   This secret will be fetched from the leverege-docker-images cluster and
+   injected into this cluster.
 NEED_MANAGED_SECRET
-    yesToContinue "to copy and inject the cloudflare secret"
+    #yesToContinue "to copy and inject the cloudflare secret"
     TOKEN=$(getManagedSecret "CLOUDFLARE_API_TOKEN")
     if [ $? -ne 0 ];
     then
@@ -501,7 +437,8 @@ DELAY
   sleep 20
   kubectl -n cert-manager apply -f cert-manager
 
-  if [[ $? -eq 1 ]]; then
+  if [[ $? -eq 1 ]];
+  then
 
     `color y "***WARNING: cert-manager components still coming up"`
 
@@ -518,19 +455,19 @@ HINTS
   removeHelmRepo jetstack
 }
 
-function installTraefik() {
-  showInstalling "Traefik Load Balancer / Router"
+function installTraefikEnvironment() {
+  showInstalling "Traefik Namespace and Cloudflare Secret"
   createNamespaceIfNeeded traefik
 
   kubectl get secret cloudflare --namespace traefik &>/dev/null
-  if [[ $? -eq 1 ]]; then
+  if [[ $? -eq 1 ]];
+  then
     cat<<NEED_TRAEFIK_SECRET
    $RED_ERROR missing secret: `color y cloudflare` namespace: `color y traefik`
 
-   This secret may be fetched from the leverege-docker-images cluster and
-   properly injected into this cluster.
+   Injecting the `color y cloudflare` secret from the `color g leverege-docker-images` cluster.
+
 NEED_TRAEFIK_SECRET
-    yesToContinue "to copy and inject the cloudflare secret"
     TOKEN=$(getManagedSecret "CLOUDFLARE_API_TOKEN")
     if [ $? -ne 0 ];
     then
@@ -542,65 +479,25 @@ NEED_TRAEFIK_SECRET
       --namespace traefik \
       --from-literal=dns-token=$TOKEN
   fi
-
-  TRAEFIK_DIR="traefik"
-  [ ! -d "$TRAEFIK_DIR" ] && mkdir $TRAEFIK_DIR
-  TRAEFIK_HA_YAML="$TRAEFIK_DIR/traefik-ha-values.yaml"
-  if [ ! -f "$TRAEFIK_HA_YAML" ];
-  then
-    touch "$TRAEFIK_DIR/.nohelm"
-    cat<<TRAEFIK_CFG>>"$TRAEFIK_HA_YAML"
-additionalArguments:
-  - "--accesslog=true"
-  - "--accesslog.format=json"
-  - "--log.level=WARN"
-  - "--metrics.prometheus=true"
-  - "--api.dashboard=true"
-
-deployment:
-  replicas: 3
-
-#service:
-#  spec:
-#   loadBalancerIP: xx.xx.xx.xx
-TRAEFIK_CFG
-  fi
-
-  addHelmRepo traefik https://helm.traefik.io/traefik
-
-  [ -z "$TRAEFIK_HELM_CHART" ] && TRAEFIK_HELM_CHART="16"
-  helm upgrade --install traefik traefik/traefik \
-    --namespace traefik \
-    --version  $TRAEFIK_HELM_CHART \
-    --values   $TRAEFIK_HA_YAML
- 
-  removeHelmRepo traefik
 }
 
 function installVelero() {
   # Follows instructions => https://github.com/vmware-tanzu/velero-plugin-for-gcp
   showInstalling "Velero Backup Management"
 
-  ## Set the project id by parsing current context
-  currContext=`kubectl config current-context`
-   
-  IFS='_'
-  read -ra ADDR <<< "$currContext" 
-
-  PROJECT_ID=${ADDR[1]} # project id is the second string split by _
-
-  gcloud config set project $PROJECT_ID
+  gcloud config set project $GCP_PROJECT_ID
 
   ## Create a bucket
-  BUCKET="$PROJECT_ID-velero"
-  printf "\nCreating storage bucket...\n"
-  gsutil mb gs://$BUCKET/ &> /dev/null
+  BUCKET="$GCP_PROJECT_ID-velero"
+  GCE_REGION=`overwhelm -k GCE_REGION`
+  printf "\nCreating storage bucket `color g $BUCKET` in `color g $GCE_REGION`\n"
+  gsutil mb -l $GCE_REGION gs://$BUCKET/ &> /dev/null
   warnOnError $? "Bucket failed to create or already exists - go check it\n"
 
   ## Create a service account
   printf "\nCreating the service account\n"
   gcloud iam service-accounts create velero \
-    --project $PROJECT_ID \
+    --project $GCP_PROJECT_ID \
     --display-name "Velero service account" \
     --description "Velero service account" &>/dev/null
   if [ $? -eq 0 ];
@@ -611,7 +508,7 @@ function installVelero() {
     printf "$YELO_WARN Failed to create the service account - does it already exist?"
   fi
 
-  SERVICE_ACCOUNT_EMAIL="velero@${PROJECT_ID}.iam.gserviceaccount.com"
+  SERVICE_ACCOUNT_EMAIL="velero@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
 
   ## Attach roles
   ROLE_PERMISSIONS=(
@@ -624,6 +521,7 @@ function installVelero() {
     compute.snapshots.delete
     compute.zones.get
     iam.serviceAccounts.actAs
+    iam.serviceAccounts.signBlob
     storage.objects.create
     storage.objects.delete
     storage.objects.get
@@ -632,7 +530,7 @@ function installVelero() {
 
   printf "\nAssigning roles to the SA\n"
   gcloud iam roles create velero.server \
-    --project $PROJECT_ID \
+    --project $GCP_PROJECT_ID \
     --title "Velero Server" \
     --permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &>/dev/null
   if [ $? -ne 0 ];
@@ -640,23 +538,23 @@ function installVelero() {
     # assume error is that role exists so try updating instead
     printf "\nUpdating roles on the SA\n"
     gcloud iam roles update velero.server \
-      --project $PROJECT_ID \
+      --project $GCP_PROJECT_ID \
       --title "Velero Server" \
       --permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &>/dev/null
   fi
   warnOnError $? "Failed to add roles to velero.server - perhaps they already exist?"
 
   printf "\nBinding IAM policy to the SA\n"
-  gcloud projects add-iam-policy-binding $PROJECT_ID \
-    --project $PROJECT_ID \
+  gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
+    --project $GCP_PROJECT_ID \
     --member serviceAccount:$SERVICE_ACCOUNT_EMAIL \
     --condition 'None' \
-    --role projects/$PROJECT_ID/roles/velero.server &>/dev/null
+    --role projects/$GCP_PROJECT_ID/roles/velero.server &>/dev/null
   warnOnError $? "gcloud problem binding IAM policy"
 
   printf "\nBinding workload identity IAM policy to the SA\n"
   gcloud iam service-accounts add-iam-policy-binding \
-      --member "serviceAccount:$PROJECT_ID.svc.id.goog[velero/velero]" \
+      --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[velero/velero]" \
       --role roles/iam.workloadIdentityUser \
       $SERVICE_ACCOUNT_EMAIL
   warnOnError $? "gcloud workload identity binding may have failed"
@@ -677,26 +575,29 @@ function installVelero() {
   gsutil iam ch serviceAccount:$SERVICE_ACCOUNT_EMAIL:objectAdmin gs://$BUCKET
 
   ## Clean up from previous failed install
-  kubectl delete volumesnapshotlocation.velero.io -n velero  gcp &>/dev/null
+  kubectl delete volumesnapshotlocation.velero.io -n velero gcp &>/dev/null
 
-  ## Install velero chart   => https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml
-  ## Helpful debugging link => https://github.com/vmware-tanzu/helm-charts/issues/351
+  ## Install velero chart      => https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml
+  ## The velero plugin for GCP => https://github.com/vmware-tanzu/velero-plugin-for-gcp
+  ## Helpful debugging link    => https://github.com/vmware-tanzu/helm-charts/issues/351
   addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
-  [ -z "$VELERO_HELM_CHART" ] && VELERO_HELM_CHART="2"
+  ## Build velero command
+  [ -z "$VELERO_HELM_CHART" ] && VELERO_HELM_CHART="3"
   helm upgrade --install velero vmware-tanzu/velero \
-    --version $VELERO_HELM_CHART \
-    --namespace velero \
+    --version "$VELERO_HELM_CHART" \
+    --namespace "velero" \
     --set credentials.useSecret="false" \
     --set configuration.provider="gcp" \
-    --set configuration.backupStorageLocation.name=default \
+    --set configuration.backupStorageLocation.name="default" \
     --set configuration.backupStorageLocation.bucket="${BUCKET}" \
-    --set configuration.backupStorageLocation.config.serviceAccount=$SERVICE_ACCOUNT_EMAIL \
+    --set configuration.backupStorageLocation.config.serviceAccount="$SERVICE_ACCOUNT_EMAIL" \
     --set serviceAccount.server.create="false" \
     --set serviceAccount.server.name="velero" \
     --set snapshotsEnabled="true" \
-    --set configuration.volumeSnapshotLocation.name=default \
+    --set configuration.volumeSnapshotLocation.name="default" \
+    --set configuration.volumeSnapshotLocation.config.snapshotLocation="$GCE_REGION" \
     --set "initContainers[0].name"="velero-plugin-for-gcp" \
-    --set "initContainers[0].image"="velero/velero-plugin-for-gcp:v1.0.0" \
+    --set "initContainers[0].image"="velero/velero-plugin-for-gcp:v1.5.0" \
     --set "initContainers[0].volumeMounts[0].mountPath"="/target" \
     --set "initContainers[0].volumeMounts[0].name"="plugins" \
     --set schedules.postgresql.schedule="0 2 * * *" \
@@ -708,15 +609,31 @@ function installVelero() {
     --set schedules.timescale.template.ttl="720h" \
     --set schedules.timescale.template.labelSelector.matchLabels.release="timescale-db" \
     --set schedules.timescale.template.labelSelector.matchLabels.app="postgresql" \
-    --set schedules.timescale.template.snapshotVolumes="true"
+    --set schedules.timescale.template.snapshotVolumes="true" \
+    --set schedules.redis.schedule="*/30 * * * *" \
+    --set schedules.redis.template.ttl="24h" \
+    --set schedules.redis.template.labelSelector.matchLabels.name="redis" \
+    --set schedules.redis.template.snapshotVolumes="true"
 
   velero get backups
   cat<<DONE_VELERO
 
   Velero docs => `color g 'open https://velero.io/docs'`
+
+  Wait at least 1 minute before kicking off the following backups:
+
+    `color g 'velero backup create --from-schedule velero-postgresql'`
+    `color g 'velero backup create --from-schedule velero-timescale'`
+
+  This will create initial backups from the defined schedules which should
+  avoid alerts from prometheus about backups failing on new installations.
+
+  The status of the backups may be checked by running:
+
+    `color g 'overwhelm && velero get backups'`
+
 DONE_VELERO
   removeHelmRepo vmware-tanzu
-  echo -n "npm run ahoy && velero get backups" | pbcopy
 }
 
 # If a helmup.plugin file exists and is executable helmup assumes it is
@@ -737,12 +654,12 @@ PLUGIN_NOEXEC
   fi
 
   printf "\n***Executing local plugin code => `color y $PLUGIN`\n\n" && sleep 2
+  export HELM_WHAT
   . $PLUGIN # executing with . allows the plugin to use functions like color
   return 0
 }
 
 function getServiceAndChartVersion() {
-
   SERVICE=
   CHARTVER=
   FROM_VERSIONS=
@@ -806,7 +723,7 @@ function doInstall() {
     [ ! -z "$CHARTVER" ] && CHARTVER="--version $CHARTVER"
 
     # set the dry-run flag on what
-    [ "`basename $0`" == "helmwhat" ] && WHAT="--dry-run --debug"
+    [ "`basename $0`" == "helmwhat" ] && HELM_WHAT="--dry-run --debug"
 
     # If we're referencing a plugin then execute it and continue
     ifPluggedIn $SERVICE && continue
@@ -821,7 +738,8 @@ function doInstall() {
         ;;
 
       "cert-manager")
-        installCertManager
+        #installCertManager
+        bootstrapLocalSetup cert-manager
         ;;
 
       "cronZombieKiller"|"zombie-killer")
@@ -841,7 +759,7 @@ function doInstall() {
         ;;
 
       "grafana")
-        installGrafana
+        bootstrapLocalSetup grafana
         ;;
 
       "monitoring")
@@ -878,12 +796,13 @@ function doInstall() {
 
       transponder-*)
         # transponders use the same chart but different deployment name
-        HELMUP="helm upgrade --install $SERVICE leverege/transponder --values $SERVICE/values.yaml $CHARTVER $WHAT"
+        HELMUP="helm upgrade --install $SERVICE leverege/transponder --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
         doHelmup "$HELMUP"
         ;;
 
       "traefik")
-        installTraefik
+        installTraefikEnvironment
+        bootstrapLocalSetup traefik
         ;;
 
       "velero")
@@ -929,7 +848,7 @@ NO_SERVICE_MAN
           MUSEUM=$(<${OVERRIDE})
           printf "\n$YELO_WARN Local chartmuseum name override from ${SERVICE} to ${MUSEUM}\n"
         fi
-        HELMUP="helm upgrade --install $SERVICE leverege/$MUSEUM -f $SERVICE/values.yaml $CHARTVER $WHAT"
+        HELMUP="helm upgrade --install $SERVICE leverege/$MUSEUM -f $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
         doHelmup $HELMUP
         SAVERR=$?
         if [ $SAVERR -ne 0 ];
@@ -957,177 +876,8 @@ NOTRUNNING
 GITTOP=`git rev-parse --show-toplevel` # `&& cd $GITTOP
 K8S_UTILS="${GITTOP}/k8s-utils"
 
-overwhelm.js #XXX
+overwhelm
 [ $? -ne 0 ] && printf "\n\n***Aborting helmup...\n\n" && exit 1
 
 doInstall $@
 cd - &> /dev/null
-
-# *** DEPRECATED *** DEPRECATED *** DEPRECATED *** DEPRECATED *** DEPRECATED *** DEPRECATED ***
-function installPrometheusXXX() {
-  # Add the community helm repositories
-  addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
-  [ -z "$PROMETHEUS_HELM_CHART" ] && PROMETHEUS_HELM_CHART="15"
-  helm upgrade --install prometheus prometheus-community/prometheus \
-    --namespace monitoring \
-    --values prometheus/values.yaml \
-    --version $PROMETHEUS_HELM_CHART
-  removeHelmRepo prometheus-community
-}
-
-function installRedisXXX() {
-  showInstalling "Redis Network Cache"
-  addBitnamiRepo latest
-  [ -z "$REDIS_HELM_CHART" ] && REDIS_HELM_CHART="17" # latest chart 17 is redis v7
-  helm upgrade --install redis bitnami/redis \
-    --version $REDIS_HELM_CHART \
-    -f - <<REDIS_CHART_MODS
-global:
-  storageClass: ssd
-
-auth:
-  enabled: false
-  sentinel: false
-
-master:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: target-env
-            operator: In
-            values:
-            - database
-
-  tolerations:
-  - key: "database"
-    operator: "Equal"
-    value: "true"
-    effect: "NoSchedule"
-
-  persistence:
-    size: 1Gi # 8Gi is default
-
-replica:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: target-env
-            operator: In
-            values:
-            - database
-
-  tolerations:
-  - key: "database"
-    operator: "Equal"
-    value: "true"
-    effect: "NoSchedule"
-
-  persistence:
-    size: 8Gi # 8Gi is default
-
-architecture: replication
-
-sentinel:
-  enabled: false
-
-networkPolicy:
-  enabled: true
-  allowExternal: false
-  ingressNSMatchLabels:
-    redis: external
-  ingressNSPodMatchLabels:
-    redis-client: true
-
-metrics:
-  enabled: true
-
-REDIS_CHART_MODS
-  if [[ $? -ne 0 ]];
-  then
-    clearlyWarn noclear
-    cat<<OLDER_REDIS
-  The redis upgrade failed which may mean that the versions are different
-  enough that k8s cannot gracefully upgrade from one version to the next.
-  Try removing the existing deployment and redeploying redis:
-
-     $ `color r "helmdn redis && sleep 5 && helmup redis"`
-OLDER_REDIS
-  fi
-  printf "\n\nTo start a specific version => `color g 'REDIS_HELM_CHART="13.0.1" helmup redis'`\n\n"
-  removeHelmRepo bitnami
-}
-
-function installPostgreSQLXXX() {
-  showInstalling "PostgreSQL Database"
-  addBitnamiRepo
-  helm upgrade --install postgres bitnami/postgresql -f postgres/postgres-values.yaml --version 10.16.2
-  removeHelmRepo bitnami
-}
-
-function checkTimescaleLocalValuesXXX() {
-  local TSDIR="$GITTOP/timescale-db"
-  local TSLCL="$TSDIR/tsdb-values.yaml"
-  [ ! -d "$TSDIR" ] && mkdir "$TSDIR" && touch "$TSDIR/.nohelm"
-  [ ! -f "$TSLCL" ] && cat > $TSLCL <<TIMESCALE_LOCAL_CHART
-postgresql:
-  postgresqlExtendedConf:
-    maxWorkerProcesses: 8
-    maxParallelWorkers: 8
-    maxConnections: 500
-    sharedBuffers: 2GB
-    effectiveCacheSize: 6GB
-    maintenanceWorkMem: 1GB
-    workMem: 20MB
-    idle_in_transaction_session_timeout: 30000 # 30ms timeout
-    maxWalSize: 2GB
-    walKeepSegments: 64
-
-  resources:
-    requests:
-      memory: 4Gi
-      cpu: 1000m
-
-  persistence:
-    size: 200Gi
-
-  livenessProbe:
-    enabled: true # false for repairs
-    initialDelaySeconds: 30
-    periodSeconds: 10
-    timeoutSeconds: 5
-    failureThreshold: 6
-    successThreshold: 1
-
-  readinessProbe:
-    enabled: true
-    successThreshold: 1
-    initialDelaySeconds: 5
-    timeoutSeconds: 5
-    periodSeconds: 10
-    failureThreshold: 6
-# These values should be used when attempting to rebuild the slave VPCs
-# following a failure, typically caused by WAL problems. The failure
-# threshold of 240 with a period of 30 seconds should allow the recovery
-# process 2 hours of time for each slave VPC - lengthen as necessary.
-#   initialDelaySeconds: 60
-#   timeoutSeconds: 15
-#   periodSeconds: 30
-#   failureThreshold: 240
-TIMESCALE_LOCAL_CHART
-  echo "$TSLCL"
-}
-
-function installTimescaleDbXXX() {
-  showInstalling "Timescale Database (Leverege chart)"
-  checkTimescalePassword
-  TSDB_VAL="$(checkTimescaleLocalValues)"
-  [ -z "$TIMESCALE_HELM_CHART" ] && TIMESCALE_HELM_CHART="2"
-  helm upgrade --install timescale-db leverege/timescale-db \
-    --version $TIMESCALE_HELM_CHART \
-    -f $GITTOP/timescale-db/tsdb-values.yaml
-}
-

package/src/system-charts/cert-manager-local.yaml

@@ -0,0 +1,43 @@
+# cert-manager is disabled
+#
+# cluster-issuer.yaml.ovh
+#
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prd
+spec:
+  acme:
+    # You must replace this email address with your own.
+    # Let's Encrypt will use this to contact you about expiring
+    # certificates, and issues related to your account.
+    email: devops@leverege.com
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource used to store the account's private key.
+      name: issuer-account-key
+    # Add a single challenge solver, HTTP01 using nginx
+    solvers:
+    - dns01:
+        cloudflare:
+          email: devops@leverege.com
+          apiTokenSecretRef:
+            name: cloudflare
+            key: dns-token
+      selector:
+        dnsZones:
+        - 'OVH:<HOST>.com'
+---
+#
+# this-service.yaml.ovh
+#
+kind: Certificate
+metadata:
+  name: OVH:<PROJECT_ID>
+spec:
+  secretName: OVH:<PROJECT_ID>-cert
+  dnsNames:
+    - '*.OVH:<HOST>.com'
+  issuerRef:
+    name: letsencrypt-prd
+    kind: ClusterIssuer

package/src/system-charts/cert-manager.hup

@@ -0,0 +1,5 @@
+#!/bin/bash
+#
+# cert-manager is skipped for now - we may not really need it
+#
+skipInstalling cert-manager

package/src/system-charts/elasticsearch.hup

@@ -4,8 +4,6 @@
 #
 showInstalling "Elastic Search"
 
-checkSSDStorageClass
-
 LOCAL_ES_VALUES="elasticsearch/elasticsearch-local.yaml"
 [ ! -f "$LOCAL_ES_VALUES" ] && exitOnError 1 "Missing the local ES config file => `color y $LOCAL_ES_VALUES`"
 
@@ -13,6 +11,6 @@ addHelmRepo elastic https://helm.elastic.co
 helm upgrade --install elasticsearch elastic/elasticsearch \
   --version 7.16 \
   --values $LOCAL_ES_VALUES \
-  --set volumeClaimTemplate.storageClassName="ssd"
+  --set volumeClaimTemplate.storageClassName="ssd" $HELM_WHAT
 
 removeHelmRepo elastic

package/src/system-charts/grafana-local.yaml

@@ -0,0 +1,29 @@
+service:
+  type: "NodePort"
+  port: "9000"
+
+persistence:
+  enabled: true
+  storageClassName: ssd
+
+plugins:
+  - grafana-piechart-panel
+  - agenty-flowcharting-panel
+
+deploymentStrategy:
+  type: Recreate
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: target-env
+          operator: In
+          values:
+          - database
+tolerations:
+- key: "database"
+  operator: "Equal"
+  value: "true"
+  effect: "NoSchedule"

package/src/system-charts/grafana.hup

@@ -9,7 +9,7 @@ addHelmRepo grafana https://grafana.github.io/helm-charts
 [ -z "$GRAFANA_HELM_CHART" ] && GRAFANA_HELM_CHART="6"
 helm upgrade --install grafana grafana/grafana \
   --namespace monitoring \
-  --values grafana/values-grafana.yaml \
-  --version $GRAFANA_HELM_CHART
+  --values grafana/grafana-local.yaml \
+  --version $GRAFANA_HELM_CHART $HELM_WHAT
 
 removeHelmRepo grafana

package/src/system-charts/postgres.hup

@@ -2,10 +2,12 @@
 #
 showInstalling "PostgreSQL Database"
 
+errorIfMissingSecret authz-postgres
+
 addBitnamiRepo
 
 helm upgrade --install postgres bitnami/postgresql  \
              --values  postgres/postgres-local.yaml \
-             --version 10
+             --version 10 $HELM_WHAT
 
 removeHelmRepo bitnami

package/src/system-charts/prometheus-local.yaml

@@ -784,3 +784,9 @@ server:
 
 pushgateway:
   enabled: false
+
+# temp set to false if upgrade errors with something like:
+#   Error: UPGRADE FAILED: cannot patch "prometheus-kube-state-metrics" ... field is immutable
+#
+kubeStateMetrics:
+  enabled: true

package/src/system-charts/prometheus.hup

@@ -10,6 +10,6 @@ addHelmRepo prometheus-community https://prometheus-community.github.io/helm-cha
 helm upgrade --install prometheus prometheus-community/prometheus \
              --namespace monitoring \
              --values prometheus/values.yaml \
-             --version $PROMETHEUS_HELM_CHART
+             --version $PROMETHEUS_HELM_CHART $HELM_WHAT
 
 removeHelmRepo prometheus-community

package/src/system-charts/redis-local.yaml

@@ -23,7 +23,7 @@ master:
     effect: "NoSchedule"
 
   persistence:
-    size: 1Gi # 8Gi is default
+    size: 8Gi # 8Gi is default
 
 replica:
   affinity:

package/src/system-charts/redis.hup

@@ -7,6 +7,6 @@ addBitnamiRepo latest
 [ -z "$REDIS_HELM_CHART" ] && REDIS_HELM_CHART="17" # latest chart 17 is redis v7
 helm upgrade --install redis bitnami/redis    \
              --values  redis/redis-local.yaml \
-             --version $REDIS_HELM_CHART
+             --version $REDIS_HELM_CHART $HELM_WHAT
 
 removeHelmRepo bitnami

package/src/system-charts/timescale-db.hup

@@ -2,9 +2,9 @@
 #
 showInstalling "Timescale Database (Leverege chart)"
 
-checkTimescalePassword
+errorIfMissingSecret postgresql-password
 
 [ -z "$TIMESCALE_HELM_CHART" ] && TIMESCALE_HELM_CHART="2"
 helm upgrade --install timescale-db leverege/timescale-db   \
              --values  timescale-db/timescale-db-local.yaml \
-             --version $TIMESCALE_HELM_CHART
+             --version $TIMESCALE_HELM_CHART $HELM_WHAT

package/src/system-charts/traefik-local.yaml

@@ -0,0 +1,13 @@
+additionalArguments:
+  - "--accesslog=true"
+  - "--accesslog.format=json"
+  - "--log.level=WARN"
+  - "--metrics.prometheus=true"
+  - "--api.dashboard=true"
+
+deployment:
+  replicas: 3
+
+#service:
+#  spec:
+#   loadBalancerIP: xx.xx.xx.xx

package/src/system-charts/traefik.hup

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+showInstalling "Traefik Load Balancer / Router"
+
+#TRAEFIK_NAMESPACE="traefik-canary"
+TRAEFIK_NAMESPACE="traefik"
+createNamespaceIfNeeded $TRAEFIK_NAMESPACE
+
+addHelmRepo traefik https://helm.traefik.io/traefik
+
+[ -z "$TRAEFIK_HELM_CHART" ] && TRAEFIK_HELM_CHART="16"
+helm upgrade --install traefik traefik/traefik \
+             --namespace traefik \
+             --values  traefik/traefik-local.yaml \
+             --version $TRAEFIK_HELM_CHART $HELM_WHAT
+
+removeHelmRepo traefik