@leverege/build-tools 1.8.4

package/lib/server/helmdn

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Simple ANSI color coded string generator.
+color() {
+  local color=
+  case "$1" in
+    'g') color="92m" ;; # green
+    'c') color="96m" ;; # lt blue
+    'y') color="93m" ;; # yellow
+    'r') color="91m" ;; # red
+    'toast') color="1;3;7;91;103;5m" ;;
+  esac
+  printf "\e[$color$2\e[0m"
+}
+
+clearlyWarn() {
+  clear
+  cat<<CLEARWARN
+
+ `color toast '*** WARNING *** WARNING *** WARNING *** WARNING *** WARNING *** WARNING ***'`
+
+CLEARWARN
+}
+
+clearlyWarn
+cat<<ABOUT2NUKE
+  You are about to start nuking deployments, as in delete with --purge. Are
+  you certain this is what you want to do?
+
+  Enter "yes" if that is in fact the case.
+
+ABOUT2NUKE
+
+read ANS && [ "$ANS" != "yes" ] && exit 1
+
+#
+# This script is used to ease the pain of transition testing to chart museum
+# prior to setting up keel for new clusters / installs.
+#
+PLATFORM=(
+  api-server
+  db-curator
+  emailer
+  imagine
+  message-processor
+  messenger
+  reason
+  scheduler
+  transponder-bq
+  transponder-fb
+  transponder-sql
+)
+
+SYSTEM=(
+  descheduler
+  elastic
+  monitoring
+  redis
+)
+
+GITTOP=`git rev-parse --show-toplevel`
+cd $GITTOP
+
+npm run ahoy
+[ $? -ne 0 ] && printf "\n\n***Aborting helmup...\n\n" && exit 1
+
+helm repo update && helm search chartmuseum
+
+function doNuking() {
+
+  for PARAM in "$@";
+  do
+    SERVICE=${PARAM%/} # strip trailing / if it's there
+
+    if [ "$SERVICE" == "platform" ];
+    then
+      doNuking "${PLATFORM[@]}" && exit $?
+    fi
+
+    if [ "$SERVICE" == "ingress" ];
+    then
+      clearlyWarn
+      cat<<NOGRESS
+
+  Deleting the ingress is probably the most costly deployment to nuke since
+  it takes so long to rebuild and stabilize. Therefore, if you REALLY want
+  to do that then you can copy and paste these commands by hand and do it
+  yourself:
+
+         npm run ahoy; helm delete --purge $SERVICE
+
+NOGRESS
+      continue
+    fi
+
+    [ "$SERVICE" == "elastic" ] && doNuking elasticsearch && continue
+
+    helm delete --purge $SERVICE
+  done
+}
+
+doNuking $@
+cd - &> /dev/null

package/lib/server/helmify

@@ -0,0 +1,289 @@
+#!/bin/bash
+
+# Check if Helm is installed first
+if ! [ -x "$(command -v helm)" ]; then
+  echo 'Error: helm is not installed.'
+  echo 'To install: brew install kubernetes-helm' 
+  exit 1
+fi
+
+# Check if Chart Museum is installed
+if ! [ -x "$(command -v chartmuseum)" ]; then
+  echo 'Error: chartmuseum is not installed.'
+  echo 'To install: see instructions at https://leverege.atlassian.net/wiki/spaces/PLAT/pages/679542800/Helm+Charts+via+ChartMuseum' 
+  exit 1
+fi
+
+# Check if Chart Museum is running
+url='http://localhost:8080/'
+status_code=$(curl --write-out %{http_code} --silent --output /dev/null ${url})
+
+if [[ "$status_code" -ne 200 ]] ; then
+  echo 'Error: Chart Museum is not running'
+  echo 'To run: chartmuseum --debug --port=8080 --storage="google" --storage-google-bucket="leverege-helm-charts" --storage-google-prefix=""'
+  exit 1 
+fi
+
+# Bootstrap Helm charts
+chartname=${PWD##*/}
+chartname_go=${chartname//-/_}
+
+if [ -d helm ];
+then
+	printf "\nThere is an existing Helm repo, do you want to override?\n\n"
+	printf "Hit return to continue or ^C to exit\n\n"; read ANS
+	rm -rf helm
+fi
+
+createService() {
+  cat<<SVC_YAML > service.yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: $chartname
+spec:
+  type: NodePort
+  selector:
+    app: $chartname
+  ports:
+  - protocol: TCP
+    port: {{ .Values.$chartname_go.SERVER_PORT }}
+    targetPort: api
+SVC_YAML
+}
+
+createIngress(){
+  cat<<INGRESS_YAML > ingress.yaml
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: $chartname-ingress
+  annotations:
+    kubernetes.io/ingress.global-static-ip-name: $chartname-ingress
+spec:
+  tls:
+  - secretName: frontendtls
+  rules:
+  - host: {{ quote .Values.$chartname_go.INGRESS_HOST }}
+    http:
+      paths:
+      - backend:
+          serviceName: $chartname
+          servicePort: {{ .Values.$chartname_go.SERVER_PORT }}
+  - http:
+      paths:
+      - backend:
+          serviceName: $chartname
+          servicePort: {{ .Values.$chartname_go.SERVER_PORT }}
+INGRESS_YAML
+}
+
+# Make chart directory
+mkdir helm
+mkdir helm/templates
+
+# Make Chart.yaml
+cd helm
+
+PACKAGE_VERSION=$(node -p "require('../package.json').version")
+
+cat<<CHARTYAML > Chart.yaml
+name: $chartname
+version: 1.0.0
+appVersion: $PACKAGE_VERSION
+
+CHARTYAML
+
+# Bootstrap values.yaml
+cat<<VALUESYAML > values.yaml
+# List your env variables here
+$chartname_go:
+  VERSION: v$PACKAGE_VERSION
+  SERVER_PORT: 8080
+  IMAGE: gcr.io/leverege-docker-images/$chartname
+
+default:
+  LOG_DIR: "/var/log/levlog"
+  LOG_LEVEL: "info"
+  MAX_CHILD_PROCESSES: "1"
+
+  MODEL_FIREBASE_SERVICE_ACCOUNT: /var/config/keys/model-data
+  MODEL_FIREBASE_DATABASE_URL: <firebase URL>
+  MODEL_FIREBASE_CONFIG: /var/config/keys/model-data-config
+
+  SECRET_KEY: key.json
+
+  PUBSUB_SERVICE_ACCOUNT: /var/config/keys/pubsub
+
+  TRANSPORT_CONFIG: "{
+    \"type\": \"pubsub\",
+    \"projectId\": \"OVH:<PROJECT_NAME>\",
+    \"keyFilename\": \"/var/config/keys/pubsub/key.json\" , 
+    \"publisher\": {
+        \"batching\": {  
+           \"maxMilliseconds\": 0,
+           \"maxMessages\": 999
+        }
+    }
+  }"
+  LOG_CONFIG: "{
+    \"type\": \"bunyan\",
+    \"name\": \"${logName}\",
+    \"level\": \"info\",
+    \"streams\": [
+      {
+        \"type\": \"rotating-file\",
+        \"period\": \"1d\",
+        \"count\": 3,
+        \"path\": \"/var/log/levlog/${logName}.log\"
+      }, { 
+        \"type\": \"stackdriver\",
+        \"logName\": \"${logName}\"
+      } 
+    ] 
+  }"
+  CACHE_CONFIG: "{
+    \"type\": \"redis\",
+    \"connection\": {
+      \"host\": \"redis-master\",
+      \"port\": \"6379\"
+    }
+  }"
+  LIMIT_CONFIG: "{
+    \"type\":\"redis\",
+    \"connection\":{
+      \"host\":\"redis-master\",
+      \"port\":6379
+    },
+    \"limitOptions\":{
+      \"frequency\":30,
+      \"period\":60000,
+      \"duration\":600000
+    }
+  }"
+  NODE_ENVIRONMENT: production
+
+VALUESYAML
+
+# Bootstrap templates 
+cd templates
+cat<<DEPLOYMENT_YAML > deployment.yaml
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: $chartname
+spec:
+  template:
+    metadata:
+      labels:
+        app: $chartname
+        redis-client: 'true'
+      annotations:
+        prometheus.io/port: '5111'
+        prometheus.io/scrape: 'true'
+    spec:
+      volumes:
+      - name: model-firebase-key
+        secret:
+          secretName: model-data
+      - name: logs
+        emptyDir: {}
+      - name: pubsub-key
+        secret:
+          secretName: pubsub
+      # UNCOMMENT TO USE PRE-EMPTIBLES
+      # tolerations:
+      # - key: "preemptible"
+      #   operator: "Equal"
+      #   value: "true"
+      #   effect: "NoSchedule"
+      # affinity:
+      #   podAntiAffinity:
+      #     preferredDuringSchedulingIgnoredDuringExecution:
+      #     - weight: 100
+      #       podAffinityTerm:   
+      #         labelSelector:
+      #           matchExpressions:
+      #           - key: app
+      #             operator: In
+      #             values:
+      #             - api-server
+      #         topologyKey: kubernetes.io/hostname
+      #   nodeAffinity:
+      #     preferredDuringSchedulingIgnoredDuringExecution:
+      #     - preference:
+      #         matchExpressions:
+      #         - key: cloud.google.com/gke-preemptible
+      #           operator: Exists
+      #       weight: 50
+      containers:
+      - name: $chartname
+        image: gcr.io/leverege-docker-images/$chartname:{{ .Values.$chartname_go.VERSION }}
+        resources:
+          requests:
+            cpu: "10m"
+        ports:
+{{- if .Values.$chartname_go.SERVER_PORT }}
+        - name: api
+          containerPort: {{ .Values.$chartname_go.SERVER_PORT }}
+{{- end }}
+        - name: node-metrics
+          containerPort: 5111
+        volumeMounts:
+        - name: model-firebase-key
+          mountPath: {{ quote .Values.default.MODEL_FIREBASE_SERVICE_ACCOUNT }}
+        - name: logs
+          mountPath: {{ quote .Values.default.LOG_DIR }}
+        - name: pubsub-key
+          mountPath: {{ quote .Values.default.PUBSUB_SERVICE_ACCOUNT }}
+        env:
+        - name: NODE_ENV
+          value: {{ .Values.default.NODE_ENVIRONMENT }}
+        - name: CACHE_CONFIG
+          value: {{ required "Cache config required" .Values.default.CACHE_CONFIG | quote }}
+        - name: LIMIT_CONFIG
+          value: {{ required "Limit config required" .Values.default.LIMIT_CONFIG | quote }}
+        - name: TRANSPORT_CONFIG
+          value: {{ required "Transport config required" .Values.default.TRANSPORT_CONFIG | quote }}
+        - name: LOG_DIR
+          value: {{ quote .Values.default.LOG_DIR }}
+        - name: LOG_LEVEL
+          value: {{ quote .Values.default.LOG_LEVEL }}
+        - name: LOG_CONFIG
+          value: {{ required "Log config required" .Values.default.LOG_CONFIG | quote }}
+        - name: MAX_CHILD_PROCESSES
+          value: {{ quote .Values.default.MAX_CHILD_PROCESSES }}
+        - name: MODEL_FIREBASE_SERVICE_ACCOUNT
+          value: {{ .Values.default.MODEL_FIREBASE_SERVICE_ACCOUNT }}/{{ .Values.default.SECRET_KEY }}
+        - name: MODEL_FIREBASE_DATABASE_URL
+          value: {{ quote .Values.default.MODEL_FIREBASE_DATABASE_URL }}
+DEPLOYMENT_YAML
+
+read -p "Does your server need an open port to the Internet (i.e. ingress or service)? [Y or N] " -n 1 -r
+echo    # (optional) move to a new line
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+  echo
+  echo '**************************************'
+  echo 'CREATING SERVICE AND INGRESS TEMPLATES'
+  echo '**************************************'
+  createService
+  createIngress
+fi
+
+# Print out some hints
+cat<<HINTS
+
+  *** IMPORTANT ***
+
+  1. You will need to add in additional env variables to both templates/*.yaml and values.yaml
+  2. You may not need all the default env variables so feel free to remove them as well
+  3. To use ingress, you need to reserve static ip as specified in ingress.yaml
+
+  	Further instructions here: https://github.com/kelseyhightower/ingress-with-static-ip
+
+  4. Finally, follow chart museum directions to add your chart or
+
+  	helm push helm/ chartmuseum
+
+HINTS

package/lib/server/helmup

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# This script is used to ease the pain of transition testing to chart museum
+# prior to setting up keel for new clusters / installs.
+#
+PLATFORM=(
+  api-server
+  db-curator
+  emailer
+  imagine
+  message-processor
+  messenger
+  reason
+  scheduler
+  transponder-bq
+  transponder-fb
+  transponder-sql
+)
+
+SYSTEM=(
+  descheduler
+  elastic
+  monitoring
+  redis
+)
+
+GITTOP=`git rev-parse --show-toplevel`
+cd $GITTOP
+
+npm run ahoy
+[ $? -ne 0 ] && printf "\n\n***Aborting helmup...\n\n" && exit 1
+
+helm repo update && helm search chartmuseum
+
+function doInstall() {
+
+  for PARAM in "$@";
+  do
+    SERVICE=${PARAM%/} # strip trailing / if it's there
+
+    case "$SERVICE" in
+      "platform")
+        doInstall "${PLATFORM[@]}"
+        ;;
+
+      "system")
+        doInstall "${SYSTEM[@]}"
+        ;;
+
+      "descheduler")
+        helm repo add akomljen-charts     https://raw.githubusercontent.com/komljen/helm-charts/master/charts/
+        helm upgrade --install descheduler akomljen-charts/descheduler --set schedule="*/13 * * * *"
+        ;;
+
+      "elastic"|"elasticsearch")
+        helm repo add elastic https://helm.elastic.co
+        helm upgrade --install elasticsearch elastic/elasticsearch --version 7.2.0
+        ;;
+
+      "monitoring")
+        helm dep up ./monitoring && helm upgrade --install monitoring ./monitoring
+        ;;
+
+      "redis")
+        # https://github.com/helm/charts/tree/master/stable/redis
+        helm upgrade --install redis stable/redis \
+          --set usePassword=false,cluster.slaveCount=3,networkPolicy.enabled=true,metrics.enabled=true
+        ;;
+
+      "ingress")
+        helm upgrade --install ingress ./ingress
+        ;;
+
+      *)
+        [ ! -d $SERVICE ] && continue
+        [ ! -f $SERVICE/values-local.yaml ] && continue
+
+        printf "\n--------------------------------------------------------------\nCharting $SERVICE\n"
+        helm upgrade --install $SERVICE chartmuseum/$SERVICE -f $SERVICE/values.yaml
+        [ $? -ne 0 ] && printf "The $SERVICE chart failed - you have issues.\n"
+        ;;
+    esac
+  done
+}
+
+doInstall $@
+cd - &> /dev/null