@leverege/build-tools 2.27.0 → 2.28.0-beta.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.27.0",
+  "version": "2.28.0-beta.2",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -55,7 +55,7 @@
     "chalk": "^4.1.2",
     "command-line-args": "^5.2.1",
     "command-line-usage": "^6.1.3",
-    "deepmerge": "^4.3.0",
+    "deepmerge": "^4.3.1",
     "dotenv": "^16.0.3",
     "enquirer": "^2.3.6",
     "execa": "^5.1.1",
@@ -66,13 +66,13 @@
     "readline-sync": "^1.4.10",
     "semver": "^7.3.8",
     "simple-git": "^3.17.0",
-    "zx": "^7.2.0"
+    "zx": "^7.2.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.21.0",
-    "@babel/core": "^7.21.0",
+    "@babel/core": "^7.21.3",
     "@leverege/babel-preset-leverege-node": "^2.0.0",
     "@leverege/eslint-config-leverege": "^3.0.1",
-    "npm": "^9.6.1"
+    "npm": "^9.6.2"
   }
 }

package/src/helm-charts/elastic-cluster/elasticsearch-client.yaml

@@ -0,0 +1,51 @@
+# client / coordinating nodes - smart load balancers
+nodeGroup: "client"
+clusterName: "elasticsearch"
+
+#esJavaOpts: "-Xmx1g -Xms1g"
+#
+resources:
+  limits:            #
+    cpu:             #  1000m default
+    memory: "2Gi"    #  2Gi
+  requests:          # values.yaml defaults
+    cpu: "200m"      #  1000m
+    memory: "1500Mi" #  2Gi
+
+volumeClaimTemplate:
+  storageClassName: "ssd"
+  resources:
+    requests:
+      storage: "1Gi"
+
+roles:    # client / coordinator
+  master: "false"
+  ingest: "false"
+  data: "false"
+  remote_cluster_client: "false"
+  ml: "false"
+
+replicas: 3
+antiAffinity: "hard" # hard is default
+
+esConfig:
+  elasticsearch.yml: |
+    xpack.security.enabled: false
+
+persistence:
+  enabled: false
+
+tolerations:
+- key: "elastic"
+  operator: "Equal"
+  value: "true"
+  effect: "NoSchedule"
+
+nodeAffinity:
+  requiredDuringSchedulingIgnoredDuringExecution:
+    nodeSelectorTerms:
+    - matchExpressions:
+      - key: target-env
+        operator: In
+        values:
+        - elastic

package/src/helm-charts/elastic-cluster/elasticsearch-data.yaml

@@ -0,0 +1,51 @@
+# 
+nodeGroup: "data"
+clusterName: "elasticsearch"
+
+esJavaOpts: "-Xmx4g -Xms4g"
+
+resources:
+  limits:         #
+    cpu:          #  1000m default
+    memory: "8Gi" #  2Gi
+  requests:       # values.yaml defaults
+    cpu: "1000m"  #  1000m
+    memory: "4Gi" #  2Gi
+
+volumeClaimTemplate:
+  storageClassName: "ssd"
+  resources:
+    requests:
+      storage: "64Gi"
+
+roles:    # data
+  master: "false"
+  ingest: "true"
+  data: "true"
+  remote_cluster_client: "false"
+  ml: "false"
+
+replicas: 3
+antiAffinity: "soft" # hard is default
+
+esConfig:
+  elasticsearch.yml: |
+    xpack.security.enabled: false
+
+persistence:
+  enabled: true
+
+tolerations:
+- key: "elastic"
+  operator: "Equal"
+  value: "true"
+  effect: "NoSchedule"
+
+nodeAffinity:
+  requiredDuringSchedulingIgnoredDuringExecution:
+    nodeSelectorTerms:
+    - matchExpressions:
+      - key: target-env
+        operator: In
+        values:
+        - elastic

package/src/helm-charts/elastic-cluster/elasticsearch-master.yaml

@@ -0,0 +1,51 @@
+# master - manages indexes, tracks nodes and shards
+nodeGroup: "master"
+clusterName: "elasticsearch"
+
+#esJavaOpts: "-Xmx1g -Xms1g"
+#
+resources:
+  limits:            #
+    cpu:             #  1000m default
+    memory: "2Gi"    #  2Gi
+  requests:          # values.yaml defaults
+    cpu: "100m"      #  1000m
+    memory: "1500Mi" #  2Gi
+
+volumeClaimTemplate:
+  storageClassName: "ssd"
+  resources:
+    requests:
+      storage: "4Gi"
+
+roles:    # master
+  master: "true"
+  ingest: "false"
+  data: "false"
+  remote_cluster_client: "false"
+  ml: "false"
+
+replicas: 3
+antiAffinity: "hard" # hard is default
+
+esConfig:
+  elasticsearch.yml: |
+    xpack.security.enabled: false
+
+persistence:
+  enabled: true
+
+tolerations:
+- key: "elastic"
+  operator: "Equal"
+  value: "true"
+  effect: "NoSchedule"
+
+nodeAffinity:
+  requiredDuringSchedulingIgnoredDuringExecution:
+    nodeSelectorTerms:
+    - matchExpressions:
+      - key: target-env
+        operator: In
+        values:
+        - elastic

package/src/helm-charts/grafana/gitignore

@@ -0,0 +1 @@
+grafana-local.yaml

package/src/{system-charts → helm-charts/grafana}/grafana-local.yaml.ovh

@@ -1,3 +1,4 @@
+# ONLY EDIT grafana-local.yaml.ovh / grafana-local.yaml is git ignored
 service:
   type: "NodePort"
   port: "9000"

package/src/helmdn

@@ -25,10 +25,9 @@ CLEARWARN
 
 clearlyWarn
 cat<<ABOUT2NUKE
-  You are about to start nuking deployments, as in delete with --purge. Are
-  you certain this is what you want to do?
+  You are about to cause helm to uninstall a service from the cluster.
 
-  Enter "yes" if that is in fact the case.
+  Enter "yes" to continue to the `color toast nuking`...
 
 ABOUT2NUKE
 

package/src/helmup

@@ -21,7 +21,6 @@ PLATFORM=(
   scheduler
   transponder-bq
   transponder-rt
-# transponder-postgres # deprecated
   transponder-tsdb
 )
 
@@ -33,7 +32,6 @@ SYSTEM=(
   preemptible-killer
   zombie-killer
   reloader
-# cert-manager # deprecated from SYSTEM 01/13/2023
   traefik
   velero
   monitoring # do monitoring last to show password
@@ -135,39 +133,18 @@ MISSING_HELMUP_PLUGIN
     errorExit "Installaion of `color y $SERVICE` failed - exiting"
   fi
 
-  TOOLS_SRC="$(build-tools --reporoot)/src"
-  SYSTEM_FILES="$(build-tools --reporoot)/src/system-charts"
-
-  case "$2" in
-    "set-values-local")
-      SYSTEM_CHART="$SYSTEM_FILES/$SERVICE-local.yaml"
-      TARGET_CHART="values-local.yaml"
-      ;;
-
-    "ovh-values-local")
-      SYSTEM_CHART="$SYSTEM_FILES/$SERVICE-local.yaml.ovh"
-      TARGET_CHART="$SERVICE-local.yaml.ovh"
-      echo "$SERVICE-local.yaml" > $SERVICE/.gitignore
-      ;;
-
-    *)
-      SYSTEM_CHART="$SYSTEM_FILES/$SERVICE-local.yaml"
-      TARGET_CHART="$SYSTEM_CHART"
-      touch $SERVICE/.nohelm
-      ;;
-  esac
-
-  if [ ! -f "$SYSTEM_CHART" ];
+  HELM_CHARTS="$(build-tools --reporoot)/src/helm-charts"
+  SERVICE_CHART="$HELM_CHARTS/$SERVICE"
+
+  if [ ! -d "$SERVICE_CHART" ];
   then
     rm -rf $SERVICE
-    errorExit "Missing $SERVICE chart => $(color y $SYSTEM_CHART)"
+    errorExit "Missing $SERVICE chart => $(color y $SERVICE_CHART)"
   fi
 
-  SYSTEM_HUP="$SYSTEM_FILES/$SERVICE.hup"
-  [ ! -f "$SYSTEM_HUP" ] && errorExit "Missing $SERVICE helmup script => $(color y $SYSTEM_HUP)"
-
-  cp $SYSTEM_CHART $SERVICE/$TARGET_CHART
-  cp $SYSTEM_HUP $SERVICE/helmup.plugin
+  cp -a $SERVICE_CHART .
+  # because npm tends to not allow .gitinore files the easy way...
+  [ -f "$SERVICE/gitignore" && mv $SERVICE/gitignore $SERVICE/.gitignore
   git add $SERVICE
   overwhelm.js --genvals
   ifPluggedIn $SERVICE
@@ -228,7 +205,7 @@ function installMonitoringEnvironment() {
 
   ## Install Stackdriver Exporter
   installStackdriverExporterEnvironment
-  bootstrapLocalSetup stackdriver-exporter set-values-local
+  bootstrapLocalSetup stackdriver-exporter
 
   # Add necessary helm repositories - note that stable is deprecated
   addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
@@ -247,10 +224,10 @@ es:
 ELASTIC_EXPORTER_CHART_MOD
 
   ## Install Prometheus
-  bootstrapLocalSetup prometheus set-values-local
+  bootstrapLocalSetup prometheus
 
   ## Install Grafana  
-  bootstrapLocalSetup grafana ovh-values-local
+  bootstrapLocalSetup grafana
 
   ## Clean up
   removeHelmRepo prometheus-community
@@ -414,65 +391,6 @@ OLDER_GKE
   fi
 }
 
-function installCertManager() {
-  showInstalling "Cert Managment"
-  createNamespaceIfNeeded cert-manager
-
-  kubectl get secret cloudflare --namespace cert-manager &>/dev/null
-  if [[ $? -eq 1 ]];
-  then
-    cat<<NEED_MANAGED_SECRET
-   $RED_ERROR missing secret: `color y cloudflare` namespace: `color y cert-manager`
-
-   This secret will be fetched from the leverege-docker-images cluster and
-   injected into this cluster.
-NEED_MANAGED_SECRET
-    #yesToContinue "to copy and inject the cloudflare secret"
-    TOKEN=$(getManagedSecret "CLOUDFLARE_API_TOKEN")
-    if [ $? -ne 0 ];
-    then
-      printf "$RED_ERROR getManagedSecret reponse=>[$TOKEN]"
-      exitOnError 1 "Failed to fetch the secret from leverege-docker-images"
-    fi
-    printf "\nInjecting=[$TOKEN] into cloudflare secret\n"
-    kubectl create secret generic cloudflare \
-      --namespace cert-manager \
-      --from-literal=dns-token=$TOKEN
-  fi
-
-  addHelmRepo jetstack https://charts.jetstack.io
-  [ -z "$CERT_MANAGER_HELM_CHART" ] && CERT_MANAGER_HELM_CHART="v1.10"
-  helm upgrade --install cert-manager jetstack/cert-manager \
-    --namespace cert-manager --set installCRDs=true \
-    --version $CERT_MANAGER_HELM_CHART
-
-  cat<<DELAY
-
-  `color y "20 second delay to allow cert-manager to stabilize before proceeding...\n"`
-
-DELAY
-
-  sleep 20
-  kubectl -n cert-manager apply -f cert-manager
-
-  if [[ $? -eq 1 ]];
-  then
-
-    `color y "***WARNING: cert-manager components still coming up"`
-
-    Try rerunning this apply again in 30 seconds:
-    `color g "kubectl -n cert-manager apply -f cert-manager"`
-  fi
-
-  cat<<HINTS
-
-  fetch certs: `color g "kubectl -n cert-manager get certificate"`
-  check certs: `color g "kubectl -n cert-manager describe certificate"`
-
-HINTS
-  removeHelmRepo jetstack
-}
-
 function installTraefikEnvironment() {
   showInstalling "Traefik Namespace and Cloudflare Secret"
   createNamespaceIfNeeded traefik
@@ -697,11 +615,6 @@ function doInstall() {
         doInstall "${SYSTEM[@]}"
         ;;
 
-      "cert-manager")
-        installCertManager
-        bootstrapLocalSetup cert-manager
-        ;;
-
       "cronZombieKiller"|"zombie-killer")
         installCronZombieKiller # eradicate the gke 1.20+ zombies
         ;;
@@ -710,8 +623,12 @@ function doInstall() {
         kubectl create -f https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/production/adapter.yaml
         ;;
 
-      "elastic"|"elasticsearch")
-        bootstrapLocalSetup elasticsearch
+      "elasticsearch") # simple 3 master cluster
+        bootstrapLocalSetup $SERVICE
+        ;;
+
+      "elastic-cluster") # complex master/data/client ES cluster
+        bootstrapLocalSetup $SERVICE
         ;;
 
       "estafette"|"preemptible-killer")
@@ -719,7 +636,7 @@ function doInstall() {
         ;;
 
       "grafana")
-        bootstrapLocalSetup grafana ovh-values-local
+        bootstrapLocalSetup $SERVICE
         ;;
 
       "monitoring")
@@ -731,16 +648,16 @@ function doInstall() {
         monitoringHelp && exit 0
         ;;
 
-      "postgres"|"postgresql")
-        bootstrapLocalSetup postgres
+      "postgres")
+        bootstrapLocalSetup $SERVICE
         ;;
 
       "prometheus")
-        bootstrapLocalSetup prometheus set-values-local
+        bootstrapLocalSetup $SERVICE
         ;;
 
       "redis")
-        bootstrapLocalSetup redis
+        bootstrapLocalSetup $SERVICE
         ;;
 
       "reloader")
@@ -752,11 +669,11 @@ function doInstall() {
 
       "stackdriver-exporter")
         installStackdriverExporterEnvironment
-        bootstrapLocalSetup stackdriver-exporter set-values-local
+        bootstrapLocalSetup $SERVICE
         ;;
 
-      "timescale"|"timescale-db")
-        bootstrapLocalSetup timescale-db
+      "timescale-db")
+        bootstrapLocalSetup $SERVICE
         ;;
 
       transponder-*)
@@ -767,16 +684,17 @@ function doInstall() {
 
       "traefik")
         installTraefikEnvironment
-        bootstrapLocalSetup traefik
+        bootstrapLocalSetup $SERVICE
         ;;
+
       "traefik-canary")
         # only valid if traefik is already installed
-        bootstrapLocalSetup traefik-canary
+        bootstrapLocalSetup $SERVICE
         ;;
 
       "velero")
         installVeleroEnvironment
-        bootstrapLocalSetup velero
+        bootstrapLocalSetup $SERVICE
         ;;
 
       *)

package/src/system-charts/cert-manager-local.yaml

@@ -1,43 +0,0 @@
-# cert-manager is disabled
-#
-# cluster-issuer.yaml.ovh
-#
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-prd
-spec:
-  acme:
-    # You must replace this email address with your own.
-    # Let's Encrypt will use this to contact you about expiring
-    # certificates, and issues related to your account.
-    email: devops@leverege.com
-    server: https://acme-v02.api.letsencrypt.org/directory
-    privateKeySecretRef:
-      # Secret resource used to store the account's private key.
-      name: issuer-account-key
-    # Add a single challenge solver, HTTP01 using nginx
-    solvers:
-    - dns01:
-        cloudflare:
-          email: devops@leverege.com
-          apiTokenSecretRef:
-            name: cloudflare
-            key: dns-token
-      selector:
-        dnsZones:
-        - 'OVH:<HOST>.com'
----
-#
-# this-service.yaml.ovh
-#
-kind: Certificate
-metadata:
-  name: OVH:<PROJECT_ID>
-spec:
-  secretName: OVH:<PROJECT_ID>-cert
-  dnsNames:
-    - '*.OVH:<HOST>.com'
-  issuerRef:
-    name: letsencrypt-prd
-    kind: ClusterIssuer

package/src/system-charts/cert-manager.hup

@@ -1,37 +0,0 @@
-#!/bin/bash
-#
-# cert-manager may be skipped for now?
-#
-skipInstalling cert-manager && exit 0
-
-addHelmRepo jetstack https://charts.jetstack.io
-[ -z "$CERT_MANAGER_CHART_VERSION" ] && CERT_MANAGER_CHART_VERSION="v1.10"
-helm upgrade --install cert-manager jetstack/cert-manager \
-  --namespace cert-manager --set installCRDs=true \
-  --version $CERT_MANAGER_CHART_VERSION
-
-cat<<DELAY
-
-  `color y "20 second delay to allow cert-manager to stabilize before proceeding...\n"`
-
-DELAY
-
-sleep 20
-kubectl -n cert-manager apply -f cert-manager
-
-if [[ $? -eq 1 ]];
-then
-
-  `color y "***WARNING: cert-manager components still coming up"`
-
-  Try rerunning this apply again in 30 seconds:
-  `color g "kubectl -n cert-manager apply -f cert-manager"`
-fi
-
-cat<<HINTS
-
-  fetch certs: `color g "kubectl -n cert-manager get certificate"`
-  check certs: `color g "kubectl -n cert-manager describe certificate"`
-
-HINTS
-removeHelmRepo jetstack