@leverege/build-tools 2.80.1 → 2.82.0

package/src/helmup.sh

@@ -37,6 +37,7 @@ PLATFORM=(
 )
 
 AUXILIARY=(
+  argocd
   fota-server
   geotile-server
   overdose
@@ -48,8 +49,15 @@ AUXILIARY=(
   vin-decoder-server
 )
 
-# zombie-killer re-enabled 20240801
+# There's a chicken / egg situation that arises with new clusters in that
+# traefik wants to enable a prometheus service monitor, but the prom operator
+# CRDs are yet to be installed - and the operator wants to define a traefik
+# route for grafana so traefik needs to exist early. Note, this is only a
+# problem on clean installs where the CRDs for those services are not yet
+# installed.
 SYSTEM=(
+  traefik
+  prom-operator
   elasticsearch8
   redis
   cnpg-operator
@@ -57,52 +65,24 @@ SYSTEM=(
   zombie-killer
   reflector
   reloader
-  traefik
   velero
   cnpg-db-psql-stack
   cnpg-db-tsdb-basic
-  prom-operator
 )
 
 SYSTEM_LEGACY=(
-  elasticsearch8
-  redis
+  elasticsearch
   postgres
   timescale-db
-  preemptible-killer
-  zombie-killer
-  reflector
-  reloader
-  traefik
-  velero
   monitoring # do monitoring last to show password
 )
 
 function isLeveregeService() {
   local answer=0
-  [[ ${PLATFORM[@]} =~ $1 || ${AUXILIARY[@]} =~ $1 || ${SYSTEM_LEGACY[@]}} =~ 1 || ${SYSTEM[@]} =~ $1 ]] && answer=1
+  [[ ${PLATFORM[@]} =~ $1 || ${AUXILIARY[@]} =~ $1 || ${SYSTEM_LEGACY[@]}} =~ $1 || ${SYSTEM[@]} =~ $1 ]] && answer=1
   echo $answer
 }
 
-# audit trail via slack
-# support for legacy bitnami charts
-function addBitnamiRepo() {
-  if [ ! -z "$BITNAMI_LATEST" ] || [ "$1" == "latest" ];
-  then
-    BITNAMI_URL="https://charts.bitnami.com/bitnami"
-  else
-    cat<<CATNAMI
-    Using the pre 2022 legacy charts from bitnami. To use latest run:
-
-        `color y "BITNAMI_LATEST=1 $HELMUP_ARGS"`
-
-CATNAMI
-    BITNAMI_URL="https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami"
-  fi
-
-  addHelmRepo bitnami $BITNAMI_URL
-}
-
 # Accepts 3 paramaters - 1 mandatory.
 #   bindWorkloadIdentity <Service Account> <Namespace> <Target Service Account>
 # where:
@@ -199,30 +179,9 @@ function setupStackBuckets() {
   makeUniformBucketWithCors geotile-server PUBLIC
 }
 
-function setupPgbouncerSecret() {
-  # pgbouncer uses a k8s secret for the user access list which is in the form
-  # of => "user" "password" - this function will create the pgbouncer secret
-  # if it does not exist by pulling the password from the PG_PASSWORD secret
-  # managed by the Google Secret Manager service.
-  USERLIST_SECRET='pgbouncer-userlist-secret'
-  kubectl get secret $USERLIST_SECRET &> $DEVNULL
-  [[ $? -eq 0 ]] && return # already exists - bail
-
-  cat<<PGB_SECRET
-
-  `color y "Creatng $USERLIST_SECRET k8s secret from secret manager PG_PASSWORD secret..."`
-
-PGB_SECRET
-  DECODED_PSQL_PASSWORD=$(gcloud secrets versions access latest --secret=PG_PASSWORD --project $GCP_PROJECT_ID)
-  exitOnError $? "Failed to fetch PG_PASSWORD from the secret manager"
-
-  kubectl create secret generic $USERLIST_SECRET --from-literal=userlist.txt="\"postgres\" \"$DECODED_PSQL_PASSWORD\""
-  exitOnError $? "Failed to create the $USERLIST_SECRET k8s secret"
-}
-
 function doPlatormPreInstallChecks() {
   # This is where we handle platform specific things that need to be setup,
-  # usuall once, and then promptly forgotten about.
+  # usually once, and then promptly forgotten about.
   setupStackBuckets
 }
 
@@ -265,11 +224,56 @@ MISSING_HELMUP_PLUGIN
   cp -a $SERVICE_CHART .
   # because npm tends to not allow .gitinore files the easy way...
   find $SERVICE -type f -name "gitignore" -execdir mv {} .gitignore \;
-  git add $SERVICE
   overwhelm --genvals
+  FIRST_RUN="$SERVICE/helmup.bootstrap"
+  if [ -f "$FIRST_RUN" ];
+  then
+    [ ! -x "$FIRST_RUN" ] && chmod +x "$FIRST_RUN" # force it to be executable
+    cat<<SERVICE_BOOTSTRAP
+  
+`color y "*** Bootstrapping $SERVICE with =>"` `color g "$FIRST_RUN"`
+
+SERVICE_BOOTSTRAP
+
+    . "$FIRST_RUN"
+    if [[ $? = 1 ]];
+    then
+      rm -rf "$SERVICE"
+      exit 1
+    fi
+  fi
+  git add $SERVICE
   ifPluggedIn $SERVICE
 }
 
+# some initial installs may be performed from the local setups in the
+# build-tools helm-charts directory
+function ifCanBootstrap() {
+  local bootstrap_targets=(
+    argocd
+    cnpg-operator
+    cnpg-db-psql-stack
+    cnpg-db-tsdb-basic
+    elasticsearch8
+    pgbouncer
+    prom-operator
+    redis
+    valkey
+    traefik
+    traefik-canary
+    velero
+  )
+  local svc_name="$1"
+
+  if [[ " ${bootstrap_targets[*]} " == *" $svc_name "* ]];
+  then
+    bootstrapLocalSetup $svc_name
+    return 0
+  else
+    return 1
+  fi
+}
+
 function installDefaultConfigs() {
   showInstalling "Default Configs"
 
@@ -298,110 +302,6 @@ function errorIfMissingSecret() {
   fi
 }
 
-function installFirebaseDatabases() {
-  showInstalling "Firebase Realtime Databases"
-
-  open https://console.firebase.google.com/u/0/project/${GCP_PROJECT_ID}/authentication/users
-  yesToContinue "when you have pressed Get Started on the Firebase Authentication page"
-
-  # create the firebase primary db
-  firebase --project $GCP_PROJECT_ID database:instances:create ${PROJECT}
-
-  # TODO: should maybe be more flexible with number of shards in the future
-  for SHARD in {0..9}
-  do
-    SHARD="${GCP_PROJECT_ID}-rt-${SHARD}"
-    printf "Creating shard $SHARD\n"
-    firebase --project $GCP_PROJECT_ID database:instances:create ${SHARD}
-  done
-}
-
-function installStackdriverExporterEnvironment() {
-  showInstalling "Stackdriver Exporter Service Account Binding"
-
-  # Add necessary helm repositories - note that stable is deprecated
-  addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
-
-  local SDEXP_NS=${1:-monitoring}
-  local SDEXP_SA="stackdriver-exporter"
-  local SDEXP_EM="$SDEXP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
-
-  createNamespaceIfNeeded $SDEXP_NS
-
-  # guarantee any previous remnants of the exporter are gone
-  printf "\n*** Removing the previous $SDEXP_SA installation...\n"
-  helm uninstall -n $SDEXP_NS $SDEXP_SA                 &> $DEVNULL
-  kubectl delete serviceaccounts -n $SDEXP_NS $SDEXP_SA &> $DEVNULL
-  printf "!!! Skipping deletion of the gcloud SA $SDEXP_EM\n"
-# gcloud --quiet iam service-accounts delete $SDEXP_EM  &> $DEVNULL
-
-  # NOTE: The k8s SA != GCP SA - the former is created by the helm chart
-  # that installs the stackdriver-exporter, the latter is handled here in
-  # order to provide the proper IAM permissions to the exporter.
-
-  # Create GCP SA and bind policies
-  printf "\n*** Creating GCP SA with $SDEXP_NS/viewer ***\n"
-  gcloud iam service-accounts create $SDEXP_SA \
-    --project "$GCP_PROJECT_ID" \
-    --display-name "Prometheus Stackdriver Exporter" &> $DEVNULL
-  warnOnError $? "the GCP SA $SDEXP_SA may already exist"
-
-  printf "\n*** Binding IAM role of $SDEXP_NS viewer to $SDEXP_SA in $GCP_PROJECT_ID\n"
-  gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \
-    --role "roles/monitoring.viewer" \
-    --member "serviceAccount:$SDEXP_EM" &> $DEVNULL
-  sleep 2 # give the IAM binding a chance to complete before moving on
-
-  bindWorkloadIdentity $SDEXP_SA $SDEXP_NS
-
-  removeHelmRepo prometheus-community
-}
-
-function installMonitoringEnvironment() {
-  showInstalling "Monitoring Components"
-  createNamespaceIfNeeded monitoring
-
-  ## Install Stackdriver Exporter
-  installStackdriverExporterEnvironment
-  bootstrapLocalSetup stackdriver-exporter
-
-  # Add necessary helm repositories - note that stable is deprecated
-  addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
-
-  ## Install Prometheus
-  bootstrapLocalSetup prometheus
-
-  ## Install Grafana  
-  bootstrapLocalSetup grafana
-
-  ## Clean up
-  removeHelmRepo prometheus-community
-}
-
-function monitoringHelp() {
-  MONPASS=`kubectl get secret --namespace monitoring grafana -o jsonpath='{.data.admin-password}' | base64 --decode`
-
-  cat<<ACCESS
-  `color g "Proxy to Prometheus (manage targets, configs, alerts):"`
-    export POD_NAME=\$(kubectl get pods --namespace monitoring -l 'app=prometheus,component=server' -o jsonpath='{.items[0].metadata.name}')
-    kubectl --namespace monitoring port-forward \$POD_NAME 9090
-
-  `color g "Proxy to Alertmanager (manage alert notification, silences):"`
-    export POD_NAME=\$(kubectl get pods --namespace monitoring -l 'app=prometheus,component=alertmanager' -o jsonpath='{.items[0].metadata.name}')
-    kubectl --namespace monitoring port-forward \$POD_NAME 9093
-
-  `color g "Proxy to Grafana (the charts you're looking for):"`
-    export GRAFANA_POD=\$(kubectl --namespace monitoring get pods -l 'app.kubernetes.io/instance=grafana' -o jsonpath='{.items[0].metadata.name}')
-    kubectl --namespace monitoring port-forward \$GRAFANA_POD 3000 &
-    sleep 2; open http://localhost:3000
-
-  `color toast "Save monitoring login to Keeper"`   => `color r "admin / $MONPASS"`
-
-  `color g "Show this helpful message again with"` => `color r "helmup monitoring-wtf"`
-
-ACCESS
-}
-
 # https://github.com/estafette/estafette-gke-preemptible-killer
 function installPreemptibleKiller() {
   showInstalling "Preemptible Node Killing Coordinator"
@@ -514,34 +414,6 @@ status: {}
 END_OF_ZOMBIE
 }
 
-function installCertManagerSecret() {
-  showInstalling "Cert Managment"
-  createNamespaceIfNeeded cert-manager
-
-  local NSPACE="default"
-  kubectl get secret cloudflare --namespace $NSPACE &> $DEVNULL
-  if [[ $? -eq 1 ]];
-  then
-    cat<<NEED_MANAGED_SECRET
-   $RED_ERROR missing secret: `color y cloudflare` namespace: `color y $NSPACE`
-
-   This secret will be fetched from the leverege-docker-images cluster and
-   injected into this cluster.
-NEED_MANAGED_SECRET
-    #yesToContinue "to copy and inject the cloudflare secret"
-    TOKEN=$(getManagedSecret "CLOUDFLARE_API_TOKEN")
-    if [ $? -ne 0 ];
-    then
-      printf "$RED_ERROR getManagedSecret reponse=>[$TOKEN]"
-      exitOnError 1 "Failed to fetch the secret from leverege-docker-images"
-    fi
-    printf "\nInjecting=[$TOKEN] into cloudflare secret\n"
-    kubectl create secret generic cloudflare \
-      --namespace $NSPACE \
-      --from-literal=dns-token=$TOKEN
-  fi
-}
-
 # See => https://github.com/emberstack/kubernetes-reflector
 function reflectSecretIntoNamespaces() {
   local SECRET="$1"
@@ -566,238 +438,8 @@ metadata:
 REFLECTED_SECRET
 }
 
-# This function is used to randomly generate the postgresql database passwords
-# for the postgres and imagine users in a cnpg operator configuration. Since
-# the resultant passwords will live in the cnpg-operands namespace they will
-# be mirrored to the default namespace via the reflector service. Lastly, the
-# service account will have workload identity bound to it in the cnpg-operands
-# namespace.
-function initializeCnpgOperand() {
-  local CNPG_SERVICE="$1"
-  local CNPG_NAMESPACE="cnpg-operands"
-  local CNPG_SVC_ACCT="$CNPG_NAMESPACE-sa"
-  local IMAGINE_PW_NAME="$CNPG_SERVICE-imagine-pw"
-  local POSTGRES_PW_NAME="$CNPG_SERVICE-postgres-pw"
-
-# TODO: we use postgres root password for access - that is bad - fix it?
-# printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
-# kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
-#       --from-literal=username="imagine" \
-#       --from-literal=password="$(generatePassword)" &> $DEVNULL
-# warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
-
-# # reflect the secret to default namespace
-# reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
-
-  printf "\nCreating postgres db password for `color g $CNPG_SERVICE`\n"
-  kubectl create secret generic -n $CNPG_NAMESPACE "$POSTGRES_PW_NAME" \
-        --from-literal=username="postgres" \
-        --from-literal=password="$(generatePassword)" &> $DEVNULL
-  warnOnError $? "the `color y $POSTGRES_PW_NAME` secret may already exist"
-
-  # reflect the secret to default namespace
-  reflectSecretIntoNamespaces $POSTGRES_PW_NAME $CNPG_NAMESPACE "default"
-
-  printf "\nBinding `color g $CNPG_SERVICE` to the `color y "$CNPG_SVC_ACCT"` SA\n"
-  bindWorkloadIdentity $CNPG_SVC_ACCT $CNPG_NAMESPACE $CNPG_SERVICE
-}
-
-function installCnpgOperatorEnvironment() {
-  # See => https://github.com/cloudnative-pg/charts
-
-  gcloud config set project $GCP_PROJECT_ID
-
-  # create an SA in the cnpg namespace for the operands to bind to
-  CNPG_NAMESPACE="cnpg-operands"
-  CNPG_SVC_ACCT="$CNPG_NAMESPACE-sa"
-  CNPG_SVC_EMAIL="$CNPG_SVC_ACCT@$GCP_PROJECT_ID.iam.gserviceaccount.com"
-
-  # We used to delete the SA when we were doing clean installs but that bit
-  # me in the ass the time I moved the operator dir out of the way to run
-  # helmwhat to populate with latest operator version. The result? The SA
-  # was deleted and recreated and the existing system lost the ability to
-  # write to GCS which resulted in WAL failures. The deletion / recreation
-  # logic was only useful in the early days of creating / deleting for
-  # initial development.
-  #
-  # Synopsis: deleting an SA from an actual non-test system is BAD NEWS
-  #           and if it happens then you will need to rebind the SA to
-  #           workload identity by hand.
-  #
-  # kubectl delete serviceaccounts -n $CNPG_NAMESPACE $CNPG_SVC_ACCT &> $DEVNULL
-  # gcloud --quiet iam service-accounts delete $CNPG_SVC_EMAIL  --project $GCP_PROJECT_ID&> $DEVNULL
-  kubectl describe serviceaccounts -n $CNPG_NAMESPACE $CNPG_SVC_ACCT &> $DEVNULL
-  if [ $? -eq 0 ];
-  then
-    cat<<SKIP_CNPG_SA_CREATION
-
-  $YELO_WARN The CNPG SA $CNPG_SVC_ACCT already exists
-
-  Deleting the SA for the purposes of upgrading the operator is going to
-  wreck someone's day (probably yours) if the desire is to simply upgrade
-  in place. To force a clean reinstall, delete both the k8s and gcp SA for
-  CNPG and try again. (see comments in helmup for more info)
-
-SKIP_CNPG_SA_CREATION
-    return
-  fi
-
-  # Create the operands namespace and annotate to allow redis access
-  createNamespaceIfNeeded $CNPG_NAMESPACE
-
-  # Create GCP SA and bind policies
-  printf "\nCreating the gcloud `color g $CNPG_SVC_ACCT` service account (SA)\n"
-  gcloud iam service-accounts create $CNPG_SVC_ACCT \
-    --project "$GCP_PROJECT_ID" \
-    --description "CNPG Operands SA" \
-    --display-name "CNPG Operands SA" &> $DEVNULL
-  warnOnError $? "the GCP SA `color y $CNPG_SVC_ACCT` may already exist"
-
-  CNPG_OPS_ROLE="cnpg.operands"
-  printf "\nCreating the `color g $CNPG_OPS_ROLE` IAM role\n"
-  ## Attach roles
-  ROLE_PERMISSIONS=(
-    iam.serviceAccounts.actAs
-    iam.serviceAccounts.get
-    iam.serviceAccounts.getAccessToken
-    iam.serviceAccounts.getOpenIdToken
-    iam.serviceAccounts.implicitDelegation
-    iam.serviceAccounts.list
-    iam.serviceAccounts.signBlob
-    iam.serviceAccounts.signJwt
-    resourcemanager.projects.get
-    storage.buckets.create
-    storage.buckets.get
-    storage.buckets.getIamPolicy
-    storage.buckets.list
-    storage.buckets.update
-    storage.objects.create
-    storage.objects.delete
-    storage.objects.get
-    storage.objects.getIamPolicy
-    storage.objects.list
-  )
-  gcloud iam roles create $CNPG_OPS_ROLE \
-    --project $GCP_PROJECT_ID \
-    --title "CNPG Operands" \
-    --description "CNPG Operands" \
-    --permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &> $DEVNULL
-  if [ $? -ne 0 ];
-  then
-    # assume error is that role exists so try updating instead
-    printf "\nUpdating the permissions on the `color g $CNPG_OPS_ROLE` IAM role\n"
-    gcloud iam roles update $CNPG_OPS_ROLE \
-      --project $GCP_PROJECT_ID \
-      --title "CNPG Operands" \
-      --permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &> $DEVNULL
-  fi
-  warnOnError $? "Failed to add roles to $CNPG_OPS_ROLE - perhaps they already exist?"
-
-  printf "\nBinding IAM role to the SA `color g $CNPG_SVC_EMAIL`\n"
-  gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
-    --member="serviceAccount:$CNPG_SVC_EMAIL" \
-    --role "roles/storage.admin" &> $DEVNULL
-#   --role projects/$GCP_PROJECT_ID/roles/$CNPG_OPS_ROLE &> $DEVNULL
-#   --role "roles/storage.admin" &> $DEVNULL
-  warnOnError $? "gcloud problem binding IAM policy"
-
-  printf "\nBinding IAM the Token Creator role to the SA `color g $CNPG_SVC_EMAIL`\n"
-  gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
-    --member="serviceAccount:$CNPG_SVC_EMAIL" \
-    --role="roles/iam.serviceAccountTokenCreator"
-
-  # workload binding will be performed by each cnpg operand on startup
-
-  BARMAN_BUCKET="gs://$GCP_PROJECT_ID-barman"
-  gsutil ls -p $GCP_PROJECT_ID -b $BARMAN_BUCKET &> $DEVNULL
-  if [ "$?" -eq 0 ];
-  then
-    printf "\nBarman bucket `color g "$BARMAN_BUCKET"` already exists - skipping recreation\n"
-  else
-    printf "\nCreating a bucket for barman backups: `color g $BARMAN_BUCKET`\n"
-    gsutil mb -l $GCE_REGION $BARMAN_BUCKET &> $DEVNULL
-    gsutil iam ch serviceAccount:$CNPG_SVC_EMAIL:objectAdmin $BARMAN_BUCKET
-    gsutil ubla set on $BARMAN_BUCKET # uniform bucket access
-  fi
-
-  # define the snapshot storage class
-  printf "\nApplying the CNPG VolumeSnapshotClass `color g cnpg-snapshotclass` definition\n"
-  cat<<EOSNAPSC | kubectl apply -f -
-apiVersion: snapshot.storage.k8s.io/v1
-kind: VolumeSnapshotClass
-metadata:
-  name: cnpg-snapshotclass
-driver: pd.csi.storage.gke.io
-deletionPolicy: Retain
-EOSNAPSC
-
-  sleep 2 # hold up processing for a moment to allow IAM mods to propagate
-}
-
-# this function is a little convoluted in the way it attempts to install the
-# scripts and yaml necessary for the CNPG cleanup activities - the idea is to
-# be able to update the scripts without touching anything else since the target
-# directory is the cnpg-operator setup
-function installCnpgCleanupCronjob() {
-  CNPG_OPERATOR_DIR="cnpg-operator"
-  if [ ! -d "$CNPG_OPERATOR_DIR" ];
-  then
-    cat<<CNPG_OPERATOR_MISSING
-  `color r "The cnpg-operator must be installed before proceeding"`
-
-CNPG_OPERATOR_MISSING
-    exit 1
-  fi
-
-  # create the destination bucket for k8s cronjob scripts
-  BUCKET_SUFFIX="k8s-cronjobs"
-  makeUniformBucketWithCors $BUCKET_SUFFIX
-
-  CNPG_CLEANUP_CRONJOB_OVH="cnpg-cleanup-cronjob.yaml.ovh"
-  CNPG_CLEANUP_CRONJOB="$CNPG_OPERATOR_DIR/$CNPG_CLEANUP_CRONJOB_OVH"
-  CNPG_CLEANUP_SCRIPT_BASH="cnpg-cleanup-script.sh"
-  CNPG_CLEANUP_SCRIPT="$CNPG_OPERATOR_DIR/$CNPG_CLEANUP_SCRIPT_BASH"
-  if [ ! -f "$CNPG_CLEANUP_SCRIPT" ];
-  then
-    cat<<CNPG_FETCH_CLEANUP_CRONJOB_YAML
-  `color y "Missing $CNPG_CLEANUP_SCRIPT_BASH - updating local cron setup scripts"`
-
-CNPG_FETCH_CLEANUP_CRONJOB_YAML
-    CNPG_OP_ROOT="$(build-tools --reporoot)/src/helm-charts/cnpg-operator"
-    cp $CNPG_OP_ROOT/$CNPG_CLEANUP_CRONJOB_OVH $CNPG_CLEANUP_CRONJOB
-    cp $CNPG_OP_ROOT/$CNPG_CLEANUP_SCRIPT_BASH $CNPG_CLEANUP_SCRIPT
-  fi
-
-  # copy / update the script in the bucket referenced in the cronjob
-  printf "\ngsutil cp $CNPG_CLEANUP_SCRIPT gs://$GCP_PROJECT_ID-$BUCKET_SUFFIX\n"
-  gsutil cp $CNPG_CLEANUP_SCRIPT gs://$GCP_PROJECT_ID-$BUCKET_SUFFIX
-
-  local CNPG_NAMESPACE="cnpg-operands"
-  local CNPG_CLEANUP_SA="cnpg-cleanup-sa"
-
-  gcloud iam service-accounts create "$CNPG_CLEANUP_SA" \
-    --project="$GCP_PROJECT_ID" \
-    --description="Used for cronjob cleanup with Workload Identity" \
-    --display-name="CNPG Cleanup SA" &> $DEVNULL
-    warnOnError $? "the GCP SA $CNPG_CLEANUP_SA may already exist"
-
-  gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \
-    --project="$GCP_PROJECT_ID" \
-    --member="serviceAccount:$CNPG_CLEANUP_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
-    --role="roles/storage.objectViewer"
-
-  bindWorkloadIdentity "$CNPG_CLEANUP_SA" "$CNPG_NAMESPACE"
-
-  CNPG_CLEANUP_CRONJOB_YAML="$CNPG_OPERATOR_DIR/$(basename $CNPG_CLEANUP_CRONJOB_OVH .ovh)"
-  overwhelm -g # allow ovh -> yaml to occur
-  cat<<CNPG_CLEANUP_CRONJOB_APPLY
-  `color g "Creating cronjob =>"``color y " kubectl apply -f $CNPG_CLEANUP_CRONJOB_YAML"`
-
-CNPG_CLEANUP_CRONJOB_APPLY
-  kubectl apply -f $CNPG_CLEANUP_CRONJOB_YAML
-}
-
-function ensureCnpgSecret() {
+# uses service-man to synchronize a secret manager secret with its k8s counterpart
+function synchronizeSecrets() {
   local mgrSecret=$1
   local k8sSecret=$2
 
@@ -814,105 +456,6 @@ BAD_SECRETS
   fi
 }
 
-function installVeleroEnvironment() {
-  # Follows instructions => https://github.com/vmware-tanzu/velero-plugin-for-gcp
-  showInstalling "Velero Backup Management"
-  createNamespaceIfNeeded velero
-
-  # Do we need to => gcloud auth application-default
-  gcloud config set project $GCP_PROJECT_ID
-
-  ## The major chart version will determine the bucket suffix
-  [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="10"
-
-  ## Create a bucket
-  BUCKET="$GCP_PROJECT_ID-velero-$VELERO_CHART_VERSION"
-  printf "\nCreating storage bucket `color g $BUCKET` in `color g $GCE_REGION`\n"
-  gsutil mb -l $GCE_REGION gs://$BUCKET/ &> $DEVNULL
-  warnOnError $? "Bucket failed to create or already exists - go check it\n"
-  gsutil ubla set on gs://$BUCKET &> $DEVNULL
-
-  ## Create a service account
-  printf "\nCreating the service account\n"
-  gcloud iam service-accounts create velero \
-    --project $GCP_PROJECT_ID \
-    --display-name "Velero service account" \
-    --description "Velero service account" &> $DEVNULL
-  if [ $? -eq 0 ];
-  then
-    # no error means a new SA was created - slight delay to let it update
-    sleep 2
-  else
-    printf "$YELO_WARN Failed to create the service account - does it already exist?"
-  fi
-
-  SERVICE_ACCOUNT_EMAIL="velero@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
-
-  ## Attach roles
-  ROLE_PERMISSIONS=(
-    compute.disks.get
-    compute.disks.create
-    compute.disks.createSnapshot
-    compute.projects.get
-    compute.snapshots.get
-    compute.snapshots.create
-    compute.snapshots.useReadOnly
-    compute.snapshots.delete
-    compute.zones.get
-    iam.serviceAccounts.actAs
-    iam.serviceAccounts.signBlob
-    storage.objects.create
-    storage.objects.delete
-    storage.objects.get
-    storage.objects.list
-  )
-
-  printf "\nAssigning roles to the SA\n"
-  gcloud iam roles create velero.server \
-    --project $GCP_PROJECT_ID \
-    --title "Velero Server" \
-    --permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &> $DEVNULL
-  if [ $? -ne 0 ];
-  then
-    # assume error is that role exists so try updating instead
-    printf "\nUpdating roles on the SA\n"
-    gcloud iam roles update velero.server \
-      --project $GCP_PROJECT_ID \
-      --title "Velero Server" \
-      --permissions "$(IFS=","; echo "${ROLE_PERMISSIONS[*]}")" &> $DEVNULL
-  fi
-  warnOnError $? "Failed to add roles to velero.server - perhaps they already exist?"
-
-  printf "\nBinding IAM policy to the SA\n"
-  gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
-    --project $GCP_PROJECT_ID \
-    --member serviceAccount:$SERVICE_ACCOUNT_EMAIL \
-    --condition 'None' \
-    --role projects/$GCP_PROJECT_ID/roles/velero.server &> $DEVNULL
-  warnOnError $? "gcloud problem binding IAM policy"
-
-  printf "\nBinding workload identity IAM policy to the SA\n"
-  gcloud iam service-accounts add-iam-policy-binding \
-      --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[velero/velero]" \
-      --role roles/iam.workloadIdentityUser \
-      $SERVICE_ACCOUNT_EMAIL
-  warnOnError $? "gcloud workload identity binding may have failed"
-
-  printf "\nCreate and annotate the k8s SA\n"
-  kubectl create serviceaccount velero --namespace velero
-  warnOnError $? "Failed to create k8s SA - perhaps it already exists?"
-  
-  kubectl annotate serviceaccount velero \
-    --namespace velero \
-    "iam.gke.io/gcp-service-account=$SERVICE_ACCOUNT_EMAIL"
-  warnOnError $? "Failed to annotate k8s SA - things may fail"
-
-  gsutil iam ch serviceAccount:$SERVICE_ACCOUNT_EMAIL:objectAdmin gs://$BUCKET
-
-  ## Clean up from previous failed install
-  kubectl delete volumesnapshotlocation.velero.io -n velero gcp &> $DEVNULL
-}
-
 function getServiceAndChartVersion() {
   SERVICE=
   CHARTVER=
@@ -1050,127 +593,35 @@ CATBACKUP
     # If we're referencing a plugin then execute it and continue
     ifPluggedIn $SERVICE && continue
 
+    # If we can bootstrap a service on a new setup
+    ifCanBootstrap $SERVICE && continue
+
     case "$SERVICE" in
       "platform")
         doPlatormPreInstallChecks
         doInstall "${PLATFORM[@]}"
         ;;
 
-      "system-legacy")
-        doInstall "${SYSTEM_LEGACY[@]}"
-        ;;
-
       "system")
         doInstall "${SYSTEM[@]}"
         ;;
 
-      "cert-manager")
-        installCertManagerSecret
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      cnpg-db-*)
-        if [ ! -d $SERVICE ];
-        then
-          # this code only executes on the first deployment
-          initializeCnpgOperand $SERVICE
-          bootstrapLocalSetup $SERVICE
-        fi
-        ;;
-
-      "cnpg-operator")
-        installCnpgOperatorEnvironment
-        bootstrapLocalSetup $SERVICE
-        installCnpgCleanupCronjob
-        ;;
-
-      "cnpg-cronjob")
-        installCnpgCleanupCronjob
-        ;;
-
       "cronZombieKiller"|"zombie-killer")
         installCronZombieKiller # eradicate the gke 1.20+ zombies
         ;;
 
-      "custom-metrics")
-        kubectl create -f https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/production/adapter.yaml
-        ;;
-
       default-configs*)
         installDefaultConfigs
         ;;
 
-      "db-curator-dh")
-        # db-curator-dh is a variant of db-curator with diff controlled in values-local.yaml
-        HELMUP="helm upgrade --install $SERVICE leverege/db-curator --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
-        doHelmup "$HELMUP"
-        ;;
-
-      "elasticsearch") # simple 3 master v7.16 cluster
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "elasticsearch-cluster") # complex master/data/client ES cluster
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "elasticsearch8") # complex master/data/client ES v8 cluster
-        bootstrapLocalSetup $SERVICE
-        ;;
-
       "estafette"|"preemptible-killer")
         installPreemptibleKiller # the gke preemptible node spreader outer
         ;;
 
-      "firebase")
-        installFirebaseDatabases
-        bootstrapLocalSetup firebase-rules
-        ;;
-
-      "grafana")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "kube-prometheus-stack")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "monitoring")
-        installMonitoringEnvironment
-        monitoringHelp
-        ;;
-
-      "monitoring-wtf")
-        monitoringHelp && exit 0
-        ;;
-
       "platform-buckets")
         setupStackBuckets && exit 0
         ;;
 
-      "pgbouncer")
-        setupPgbouncerSecret
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "postgres")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "prometheus")
-        installStackdriverExporterEnvironment
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "prom-operator")
-        installStackdriverExporterEnvironment prometheus
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "redis"|"valkey")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
       "reflector")
         showInstalling "Secrets/ConfigMap Namespace Reflector"
         addHelmRepo emberstack https://emberstack.github.io/helm-charts
@@ -1185,41 +636,13 @@ CATBACKUP
         removeHelmRepo stakater
         ;;
 
-      "stackdriver-exporter")
-        installStackdriverExporterEnvironment
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "timescale-db")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
       transponder-*)
         # transponders use the same chart but different deployment name
         [ ! -d $SERVICE ] && bootstrapLocalSetup $SERVICE
         HELMUP="helm upgrade --install $SERVICE leverege/transponder --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
         doHelmup "$HELMUP"
         ;;
-
-      "traefik")
-        # installTraefikEnvironment
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "traefik-canary")
-        # only valid if traefik is already installed
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "velero")
-        installVeleroEnvironment
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "vernemq")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
+        
       *)
         # if the service dir is missing then check to see if it's a platform
         # service and if it is then try to bootstrap from charts in build-tools
@@ -1296,7 +719,7 @@ NOTRUNNING
 }
 
 # ---------- helmup MAIN ----------
-GITTOP=`git rev-parse --show-toplevel` # `&& cd $GITTOP
+GITTOP=`git rev-parse --show-toplevel`
 K8S_UTILS="${GITTOP}/k8s-utils"
 
 overwhelm
@@ -1310,5 +733,3 @@ GCE_REGION=`overwhelm -k GCE_REGION`
 doInstall $@
 
 displayBanner
-
-cd - &> $DEVNULL