@leverege/build-tools 2.8.14 → 2.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.8.14",
+  "version": "2.9.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
     "repository": {
@@ -20,6 +20,7 @@
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "bin": {
+    "anchor": "src/anchor",
     "build-tools": "src/build-tools.js",
     "circleate": "src/circleate",
     "decrypt-secrets": "src/decrypt-secrets",
@@ -31,7 +32,7 @@
     "git-tar": "src/git-tar",
     "helm-repos-reload": "src/helm-repos-reload",
     "helmdn": "src/helmdn",
-    "helmify": "lib/server/helmify/index.js",
+    "xhelmify": "lib/server/helmify/index.js",
     "helminit-dp": "src/helminit-dp",
     "helminit-scaler": "src/helminit-scaler",
     "helminit-svc": "src/helminit-svc",
@@ -56,10 +57,10 @@
     "chalk": "^4.1.2",
     "clear": "^0.1.0",
     "clipboardy": "^2.3.0",
-    "command-line-args": "^5.2.0",
+    "command-line-args": "^5.2.1",
     "command-line-usage": "^6.1.1",
     "deepmerge": "^4.2.2",
-    "dotenv": "^10.0.0",
+    "dotenv": "^14.3.2",
     "execa": "^5.1.1",
     "figlet": "^1.5.2",
     "glob": "^7.2.0",
@@ -76,10 +77,10 @@
   },
   "devDependencies": {
     "@babel/cli": "^7.16.8",
-    "@babel/core": "^7.16.7",
+    "@babel/core": "^7.16.12",
     "@babel/node": "^7.16.8",
     "@leverege/babel-preset-leverege-node": "^2.0.0",
     "@leverege/eslint-config-leverege": "^3.0.1",
-    "npm": "^8.3.1"
+    "npm": "^8.4.0"
   }
 }

package/src/helmdn

@@ -1,4 +1,5 @@
 #!/bin/bash
+# TODO: maybe update this?
 
 # Simple ANSI color coded string generator.
 color() {
@@ -55,7 +56,9 @@ PLATFORM=(
 )
 
 SYSTEM=(
+  cert-manager
   elastic
+  estafette-gke-preemptible-killer
   redis
   postgres
   timescale-db
@@ -79,15 +82,15 @@ read -ra ADDR <<< "$currContext"
 
 GCP_PROJECT_ID=${ADDR[1]} # project id is the second string split by _
 
-helm repo update # old helm2 => helm search chartmuseum
+helm repo update &> /dev/null
 
 function annihilateMonitoring() {
   kubectl delete sa -n monitoring stackdriver-exporter
   gcloud iam service-accounts delete stackdriver-exporter@${GCP_PROJECT_ID}.iam.gserviceaccount.com --quiet
 
-  helm repo remove prometheus-community kube-state-metrics && helm repo update
+  # kube-state-metrics?
   helm uninstall -n monitoring grafana prometheus stackdriver-exporter elasticsearch-exporter
-  helm ls -n monitoring
+# helm ls --all-namespaces
 }
 
 function annihilateVelero() {
@@ -112,30 +115,46 @@ function doNuking() {
   do
     SERVICE=${PARAM%/} # strip trailing / if it's there
 
-    if [ "$SERVICE" == "monitoring" ];
-    then
-      annihilateMonitoring && continue
-    fi
+    case "$SERVICE" in
+      "cert-manager")
+        helm uninstall -n cert-manager cert-manager
+        ;;
 
-    if [ "$SERVICE" == "velero-nuke" ];
-    then
-      annihilateVelero && continue
-    fi
+      "elastic"|"elasticsearch")
+        helm uninstall elasticsearch
+        ;;
 
-    if [ "$SERVICE" == "platform" ];
-    then
-      doNuking "${PLATFORM[@]}" && exit $?
-    fi
+      "estafette"|"estafette-gke-preemptible-killer")
+        helm uninstall -n estafette estafette-gke-preemptible-killer
+        ;;
 
-    if [ "$SERVICE" == "system" ];
-    then
-      doNuking "${SYSTEM[@]}" && exit $?
-    fi
+      "traefik")
+        helm uninstall -n traefik traefik
+        ;;
 
-    if [ "$SERVICE" == "ingress" ];
-    then
-      clearlyWarn
-      cat<<NOGRESS
+      "monitoring")
+        annihilateMonitoring && continue
+        ;;
+
+      "velero-nuke")
+        annihilateVelero
+        ;;
+
+      "velero")
+        printf "\nUnstalling velero requires => $(color toast 'helmdn velero-nuke')\n\n"
+        ;;
+
+      "platform")
+        doNuking "${PLATFORM[@]}" && exit $?
+        ;;
+
+      "system")
+        doNuking "${SYSTEM[@]}" && exit $?
+        ;;
+
+      "ingress")
+        clearlyWarn
+        cat<<NOGRESS
 
   Deleting the ingress is probably the most costly deployment to nuke since
   it takes so long to rebuild and stabilize. Therefore, if you REALLY want
@@ -145,12 +164,11 @@ function doNuking() {
          npm run ahoy; helm delete --purge $SERVICE
 
 NOGRESS
-      continue
-    fi
-
-    [ "$SERVICE" == "elastic" ] && doNuking elasticsearch && continue
+        ;;
 
-    helm uninstall $SERVICE
+      *)
+        helm -n default uninstall $SERVICE
+    esac
   done
 }
 

package/src/helmup

@@ -30,22 +30,15 @@ SYSTEM=(
   redis
   postgres
   timescale-db
-  monitoring
   preemptible-killer
   reloader
   cert-manager
   traefik
   velero
+  monitoring # do monitoring last to show password
 )
 
 function checkTimescalePassword(){
-  if [ -f values-databases.yaml ] || [ -d timescale-db ];
-  then
-    printf "\n\n`color r '*** Deleting old timescale files ***'`\n\n"
-    rm -rf values-databases.yaml timescale-db
-    git add -A
-  fi
-
   kubectl get secret postgresql-password
   if [[ $? -eq 1 ]]; then
     printf "\n*** Autogenerating Timescale password ***\n\n"
@@ -76,14 +69,34 @@ function createNamespaceIfNeeded(){
   fi
 }
 
+function showInstalling() {
+  cat<<INSTALLING
+
+$(color y '======================================================================')
+
+  Installing => $(color g "$1")
+
+INSTALLING
+}
+
+function addHelmRepo() {
+  local chart=$1
+  local url=$2
+
+  cat<<REPOADD
+  Adding helm repo => $(color g "$chart") from $(color g "$url")
+
+REPOADD
+  helm repo add $chart $url &> /dev/null
+  helm repo update &> /dev/null
+}
+
 function installMonitoringCharts(){
+  showInstalling "Grafana Monitoring"
   createNamespaceIfNeeded monitoring
 
   # Add necessary helm repositories - note that stable is deprecated
-  printf "\nAdding and updating helm repositories...\n"
-  helm repo add stable https://charts.helm.sh/stable &> /dev/null # deprecated REMOVE ME
-  helm repo add prometheus-community https://prometheus-community.github.io/helm-charts &> /dev/null
-  helm repo update
+  addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
   ## Set the project id by parsing current context
   currContext=`kubectl config current-context`
@@ -116,7 +129,7 @@ SA_EXISTS
       prometheus-community/prometheus-stackdriver-exporter \
       --namespace monitoring \
       --values stackdriver-exporter/values.yaml \
-      --version 1.8.0
+      --version 1.12.0
 
     # Verify the Kubernetes SA
     kubectl get sa -n monitoring $SDEXP_SA
@@ -165,15 +178,15 @@ SA_EXISTS
   helm upgrade --install elasticsearch-exporter prometheus-community/prometheus-elasticsearch-exporter \
     --namespace monitoring \
     --values elasticsearch-exporter/values-es-exporter.yaml \
-    --version 4.6
+    --version 4.11
 
   ## Install Grafana  
-  helm repo add grafana https://grafana.github.io/helm-charts && helm repo update
+  addHelmRepo grafana https://grafana.github.io/helm-charts
   helm upgrade --install grafana grafana/grafana \
     --namespace monitoring \
     --values grafana/values-grafana.yaml \
-    --version 6.11 # 6.11 last of v7 grafana
-#   --version 6.16 # latest v8 grafana
+#   --version 6.11 # 6.11 last of v7 grafana
+    --version 6.21 # latest v8 grafana
 }
 
 function monitoringHelp(){
@@ -249,9 +262,10 @@ function getProjectIdFromK8sContext() {
 
 # https://github.com/estafette/estafette-gke-preemptible-killer
 function installPreemptibleKiller() {
+  showInstalling "Preemptible Node Killing Adjusticator"
   PROJECT_ID="$(getProjectIdFromK8sContext)"
   createNamespaceIfNeeded estafette
-  helm repo add estafette https://helm.estafette.io &>/dev/null
+  addHelmRepo estafette https://helm.estafette.io
 
   gcloud iam service-accounts create preemptible-killer \
     --project=$PROJECT_ID \
@@ -276,11 +290,68 @@ function installPreemptibleKiller() {
   helm upgrade --install estafette-gke-preemptible-killer \
                estafette/estafette-gke-preemptible-killer \
                --namespace estafette \
-               --set extraEnv.BLACKLIST_HOURS="10:00-02:00"
+               --set extraEnv.BLACKLIST_HOURS="10:00-02:00" # No killing from 5AM-9PM EST
 #              --set extraEnv.WHITELIST_HOURS="12:30-13:30"
+  [ -f "google_service_account.json" ] && rm google_service_account.json
+}
+
+function installCertManager() {
+  showInstalling "Cert Managment"
+  createNamespaceIfNeeded cert-manager
+
+  kubectl get secret cloudflare --namespace cert-manager &>/dev/null
+  if [[ $? -eq 1 ]]; then
+    cat<<NEED_MANAGED_SECRET
+   $RED_ERROR missing secret: `color y cloudflare` namespace: `color y cert-manager`
+
+   This secret may be fetched from the leverege-docker-images cluster and
+   properly injected into this cluster.
+NEED_MANAGED_SECRET
+    yesToContinue "to copy and inject the cloudflare secret"
+    TOKEN=$(getManagedSecret "CLOUDFLARE_API_TOKEN")
+    if [ $? -ne 0 ];
+    then
+      printf "$RED_ERROR getManagedSecret reponse=>[$TOKEN]"
+      exitOnError 1 "Failed to fetch the secret from leverege-docker-images"
+    fi
+    printf "\nInjecting=[$TOKEN] into cloudflare secret\n"
+    kubectl create secret generic cloudflare \
+      --namespace cert-manager \
+      --from-literal=dns-token=$TOKEN
+  fi
+
+  addHelmRepo jetstack https://charts.jetstack.io
+  helm upgrade --install cert-manager jetstack/cert-manager \
+    --namespace cert-manager --set installCRDs=true \
+    --version v1.7.0
+
+  cat<<DELAY
+
+  `color y "20 second delay to allow cert-manager to stabilize before proceeding...\n"`
+
+DELAY
+
+  sleep 20
+  kubectl -n cert-manager apply -f cert-manager
+
+  if [[ $? -eq 1 ]]; then
+
+    `color y "***WARNING: cert-manager components still coming up"`
+
+    Try rerunning this apply again in 30 seconds:
+    `color g "kubectl -n cert-manager apply -f cert-manager"`
+  fi
+
+  cat<<HINTS
+
+  fetch certs: `color g "kubectl -n cert-manager get certificate"`
+  check certs: `color g "kubectl -n cert-manager describe certificate"`
+
+HINTS
 }
 
 function installTraefik() {
+  showInstalling "Traefik Load Balancer / Router"
   createNamespaceIfNeeded traefik
 
   kubectl get secret cloudflare --namespace traefik &>/dev/null
@@ -313,15 +384,10 @@ additionalArguments:
   - "--accesslog.format=json"
   - "--log.level=WARN"
   - "--metrics.prometheus=true"
-  - "--metrics.prometheus.entryPoint=metrics"
-  - "--entryPoints.metrics.address=:8082"
   - "--api.dashboard=true"
 
 deployment:
   replicas: 3
-  podAnnotations:
-    prometheus.io/port: '8082'
-    prometheus.io/scrape: 'true'
 
 #service:
 #  spec:
@@ -329,12 +395,17 @@ deployment:
 TRAEFIK_CFG
   fi
 
+  # old chart => traefik https://containous.github.io/traefik-helm-chart
+  helm repo remove traefik &> /dev/null
+  addHelmRepo traefik https://helm.traefik.io/traefik
+
   helm upgrade --install traefik traefik/traefik \
     --namespace traefik \
     --values $TRAEFIK_HA_YAML
 }
 
 function installVelero() {
+  showInstalling "Velero Backup Management"
   open https://velero.io/docs
   yesToContinue "when you are done checking the docs"
 
@@ -413,7 +484,7 @@ function installVelero() {
   kubectl delete volumesnapshotlocation.velero.io -n velero  gcp &>/dev/null
 
   ## Install velero
-  helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
+  addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
   helm upgrade --install velero vmware-tanzu/velero \
     --namespace velero \
     --set configuration.provider="gcp" \
@@ -443,46 +514,6 @@ function installVelero() {
   echo -n "npm run ahoy && velero get backups" | pbcopy
 }
 
-# ---------- helmup MAIN ----------
-GITTOP=`git rev-parse --show-toplevel` && cd $GITTOP
-if [ -d "${GITTOP}/k8-utils" ];
-then
-  cat<<BADK8
-
-$YELO_WARN The k8_utils directory should be renamed to k8s_utils
-
-  This repository has the old k8_util directory name in place. Please rename
-  it to the newer k8s_utils directory that `color c helmup` will be looking for.
-
-  Run this => `color g "git mv k8-utils k8s-utils; git add k8*utils"`
-
-$YELO_WARN Don't forget to commit and push after the renaming.
-
-BADK8
-  exit 1
-fi
-
-K8S_UTILS="${GITTOP}/k8s-utils"
-if [ ! -d "$K8S_UTILS" ];
-then
-  cat<<EOK8S
-
-$RED_ERROR Missing the k8s utils dir => `color y $K8S_UTILS`
-
-  This is a relatively new directory setup that should be copied over from one
-  of the other configuration branches. The new-baseline branch should have this
-  setup in place, so check there.
-
-EOK8S
-
-  exit 1
-fi
-
-overwhelm
-[ $? -ne 0 ] && printf "\n\n***Aborting helmup...\n\n" && exit 1
-
-helm repo update &> /dev/null # && helm search leverege
-
 # If a helmup.plugin file exists and is executable helmup assumes it is
 # an installation specific script that should be executed instead of one
 # of the standard setup scripts.
@@ -507,6 +538,9 @@ PLUGIN_NOEXEC
 
 function doInstall() {
 
+  helm remove repo stable &> /dev/null # stable was deprecated
+  helm repo update &> /dev/null
+
   for PARAM in "$@";
   do
     # Allows chart version from command line via <deployment>/<x.y.z>
@@ -539,15 +573,20 @@ function doInstall() {
         doInstall "${SYSTEM[@]}"
         ;;
 
+      "cert-manager")
+        installCertManager
+        ;;
+
       "custom-metrics")
         kubectl create -f https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/production/adapter.yaml
         ;;
 
       "elastic"|"elasticsearch")
+        showInstalling "Elastic Search"
         LOCAL_ES_VALUES="elasticsearch/es-values.yaml"
         [ ! -f "$LOCAL_ES_VALUES" ] && exitOnError 1 "Missing the local ES config file => `color y $LOCAL_ES_VALUES`"
         checkSSDStorageClass
-        helm repo add elastic https://helm.elastic.co && helm repo update
+        addHelmRepo elastic https://helm.elastic.co
         helm upgrade --install elasticsearch elastic/elasticsearch \
           --values $LOCAL_ES_VALUES \
           --set volumeClaimTemplate.storageClassName="ssd"
@@ -563,7 +602,8 @@ function doInstall() {
         ;;
 
       "postgres"|"postgresql")
-        helm repo add bitnami https://charts.bitnami.com/bitnami
+        showInstalling "PostgreSQL Database"
+        addHelmRepo bitnami https://charts.bitnami.com/bitnami
         helm upgrade --install postgres bitnami/postgresql -f postgres/postgres-values.yaml --version 8.9.6
         ;;
 
@@ -572,20 +612,23 @@ function doInstall() {
         ;;
 
       "redis")
-        # https://github.com/helm/charts/tree/master/stable/redis
-        [ -z "$REDIS_HELM_CHART" ] && REDIS_HELM_CHART="v9.1.5"
-        helm upgrade --install redis stable/redis \
-          --version $REDIS_HELM_CHART \
-          --set usePassword=false,cluster.slaveCount=3,networkPolicy.enabled=true,metrics.enabled=true
-        printf "To start a specific version => `color c 'REDIS_HELM_CHART="v9.1.5" helmup redis'`\n\n"
+        showInstalling "Redis Network Cache"
+        addHelmRepo bitnami https://charts.bitnami.com/bitnami
+        [ -z "$REDIS_HELM_CHART" ] && REDIS_HELM_CHART="13.0.1" # latest breaks us => "v14+"
+        helm upgrade --install redis bitnami/redis \
+          -f redis/redis-values.yaml \
+          --version $REDIS_HELM_CHART
+        printf "\n\nTo start a specific version => `color g 'REDIS_HELM_CHART="13.0.1" helmup redis'`\n\n"
         ;;
 
       "reloader")
-        helm repo add stakater https://stakater.github.io/stakater-charts && helm repo update
+        showInstalling "Stakater ConfigMap Reloader"
+        addHelmRepo stakater https://stakater.github.io/stakater-charts
         helm upgrade --install reloader stakater/reloader
         ;;
 
       "timescale-db")
+        showInstalling "Timescale Database (Leverege chart)"
         checkTimescalePassword
         helm upgrade --install timescale-db leverege/timescale-db
         ;;
@@ -655,6 +698,44 @@ NOTRUNNING
   printf "\nCheck helm deployment status by running => `color g helm\ ls`\n\n"
 }
 
+# ---------- helmup MAIN ----------
+GITTOP=`git rev-parse --show-toplevel` && cd $GITTOP
+if [ -d "${GITTOP}/k8-utils" ];
+then
+  cat<<BADK8
+
+$YELO_WARN The k8_utils directory should be renamed to k8s_utils
+
+  This repository has the old k8_util directory name in place. Please rename
+  it to the newer k8s_utils directory that `color c helmup` will be looking for.
+
+  Run this => `color g "git mv k8-utils k8s-utils; git add k8*utils"`
+
+$YELO_WARN Don't forget to commit and push after the renaming.
+
+BADK8
+  exit 1
+fi
+
+K8S_UTILS="${GITTOP}/k8s-utils"
+if [ ! -d "$K8S_UTILS" ];
+then
+  cat<<EOK8S
+
+$RED_ERROR Missing the k8s utils dir => `color y $K8S_UTILS`
+
+  This is a relatively new directory setup that should be copied over from one
+  of the other configuration branches. The new-baseline branch should have this
+  setup in place, so check there.
+
+EOK8S
+
+  exit 1
+fi
+
+overwhelm
+[ $? -ne 0 ] && printf "\n\n***Aborting helmup...\n\n" && exit 1
+
 doInstall $@
 cd - &> /dev/null