@leverege/build-tools 1.8.4 → 1.10.2

package/src/helmify

@@ -46,7 +46,8 @@ spec:
   selector:
     app: $chartname
   ports:
-  - protocol: TCP
+  - name: $chartname-input
+    protocol: TCP
     port: {{ .Values.$chartname_go.SERVER_PORT }}
     targetPort: api
 SVC_YAML
@@ -98,7 +99,7 @@ CHARTYAML
 cat<<VALUESYAML > values.yaml
 # List your env variables here
 $chartname_go:
-  VERSION: v$PACKAGE_VERSION
+  VERSION: vX.Y.Z
   SERVER_PORT: 8080
   IMAGE: gcr.io/leverege-docker-images/$chartname
 
@@ -108,7 +109,7 @@ default:
   MAX_CHILD_PROCESSES: "1"
 
   MODEL_FIREBASE_SERVICE_ACCOUNT: /var/config/keys/model-data
-  MODEL_FIREBASE_DATABASE_URL: <firebase URL>
+  MODEL_FIREBASE_DATABASE_URL: https://OVH:<PROJECT_NAME>.firebaseio.com
   MODEL_FIREBASE_CONFIG: /var/config/keys/model-data-config
 
   SECRET_KEY: key.json
@@ -191,34 +192,39 @@ spec:
       - name: pubsub-key
         secret:
           secretName: pubsub
-      # UNCOMMENT TO USE PRE-EMPTIBLES
-      # tolerations:
-      # - key: "preemptible"
-      #   operator: "Equal"
-      #   value: "true"
-      #   effect: "NoSchedule"
-      # affinity:
-      #   podAntiAffinity:
-      #     preferredDuringSchedulingIgnoredDuringExecution:
-      #     - weight: 100
-      #       podAffinityTerm:   
-      #         labelSelector:
-      #           matchExpressions:
-      #           - key: app
-      #             operator: In
-      #             values:
-      #             - api-server
-      #         topologyKey: kubernetes.io/hostname
-      #   nodeAffinity:
-      #     preferredDuringSchedulingIgnoredDuringExecution:
-      #     - preference:
-      #         matchExpressions:
-      #         - key: cloud.google.com/gke-preemptible
-      #           operator: Exists
-      #       weight: 50
+{{- if (.Values.$chartname_go.PREEMPTIBLE) and eq .Values.$chartname_go.PREEMPTIBLE "true" }}
+      tolerations:
+      - key: "preemptible"
+        operator: "Equal"
+        value: "true"
+        effect: "NoSchedule"
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - api-server
+              topologyKey: kubernetes.io/hostname
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: cloud.google.com/gke-preemptible
+                operator: Exists
+            weight: 50
+{{- end }}
       containers:
       - name: $chartname
         image: gcr.io/leverege-docker-images/$chartname:{{ .Values.$chartname_go.VERSION }}
+        imagePullPolicy: Always
+        securityContext:
+          runAsUser: 1000
+          allowPrivilegeEscalation: false
         resources:
           requests:
             cpu: "10m"

package/src/helmup

@@ -3,9 +3,12 @@
 # This script is used to ease the pain of transition testing to chart museum
 # prior to setting up keel for new clusters / installs.
 #
+# Removing db-curator from the list of platform components for now. It needs
+# to be turned into a cronjob to avoid having it run as soon as it gets
+# deployed.
+#
 PLATFORM=(
   api-server
-  db-curator
   emailer
   imagine
   message-processor
@@ -19,18 +22,138 @@ PLATFORM=(
 
 SYSTEM=(
   descheduler
+  disks
   elastic
   monitoring
+  polaris
   redis
 )
 
+# Simple ANSI color coded string generator.
+function color() {
+  local color=
+  case "$1" in
+    'g') color="92m" ;; # green
+    'c') color="96m" ;; # lt blue
+    'y') color="93m" ;; # yellow
+    'r') color="91m" ;; # red
+    'toast') color="1;3;7;91;103;5m" ;;
+  esac
+  printf "\e[$color$2\e[0m"
+}
+RED_ERROR=`color r '***ERROR:'`
+YELO_WARN="`color y '***WARNING:'`"
+
+function setOrUnset() {
+  local setting=`color toast "NEEDS TO BE SET"`
+  [ ! -z "$1" ] && setting=`color g $1`
+  echo "$setting"
+}
+
+# Installing keel is a little more involved since we need to feed it svc acct
+# keys and a slack bot token.
+function keelNeeds() {
+  cat<<KEELNEEDS
+
+ $RED_ERROR  $1
+
+ In order to get its job done keel needs the service account JSON file for the
+ project containing the docker images that it is reponsible for. It also needs
+ a token for the slack bot in order to interact with slack.
+
+   `color g LEV_KEYS`
+           points to the top level directory containing the keys and secrets
+           to the k8s clusters
+   `color g \$LEV_KEYS/docker-image-svc-acct`
+           directory containing the service account JSON files
+   `color g "\$LEV_KEYS/<branch-name>/slack-bot-token"`
+           location of the slack bot token text file
+
+ Current settings:
+   LEV_KEYS   => `setOrUnset $LEV_KEYS`
+
+ Missing variables should be set using the standard bash syntax:
+   `color y export\ VARNAME=\"\<YOUR_SETTING\>\"`
+
+KEELNEEDS
+  exit 1
+}
+
+function createSSDYAML(){
+  cat<<EOSSD > k8-utils/ssd-storageclass.yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ssd
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-ssd
+EOSSD
+}
+
+function checkSSDStorageClass(){
+  kubectl get storageclass ssd
+  if [[ $? -eq 1 ]]; then
+    printf "\n*** SSD Storage Class not defined ***\n\n"
+    if [ ! -f k8-utils/ssd-storageclass.yaml ]; then
+      createSSDYAML
+    fi
+    kubectl apply -f k8-utils/ssd-storageclass.yaml
+  fi
+
+  printf "\n*** Attaching PVC to SSD Class ***\n\n"
+}
+
+function installKeel() {
+  [ ! -d "$LEV_KEYS" ] &&
+    keelNeeds "`color c \\$LEV_KEYS` must point to the keys directory for the clusters."
+
+  # TODO make the docker image project configurable and not just lev
+  SVC_ACCT="$LEV_KEYS/docker-image-svc-acct/leverege-docker-images.json"
+  [ ! -f "$SVC_ACCT" ] &&
+    keelNeeds "Missing service account info => `color c $SVC_ACCT`"
+
+  BRANCH=`git rev-parse --abbrev-ref HEAD`
+  SLACK_BOT_NAME="`overwhelm -k SLACK_BOT_NAME`"
+  [ -z "$SLACK_BOT_NAME" ] && SLACK_BOT_NAME=$BRANCH
+  SLACK_TOKEN="$LEV_KEYS/$BRANCH/slack-bot-token"
+  [ ! -f "$SLACK_TOKEN" ] &&
+    keelNeeds "Missing slack bot token => `color c $SLACK_TOKEN`"
+
+  helm repo add keel https://charts.keel.sh; helm repo update
+  helm upgrade --install keel keel/keel \
+    --set gcr.enabled="true" \
+    --set gcr.projectId="leverege-docker-images" \
+    --set gcr.pubSub.enabled="true" \
+    --set slack.enabled="true" \
+    --set slack.channel="devs-cicd" \
+    --set slack.approvalsChannel="cd-approvals" \
+    --set slack.botName="$SLACK_BOT_NAME" \
+    --set slack.token="$(cat $SLACK_TOKEN)" \
+    --set googleApplicationCredentials="$(cat $SVC_ACCT | base64 -w 0)"
+}
+
+# ---------- helmup MAIN ----------
+command -v chartmuseum &> /dev/null
+if [ $? -ne 0 ];
+then
+  cat<<NOMUSEUM
+
+$RED_ERROR Missing chartmuseum. Head over to the wiki site and follow the
+           instructions in the section entitled Installing ChartMuseum down
+           in the How-to articles -> Helm Charts via ChartMuseum.
+
+NOMUSEUM
+  exit 1
+fi
+
 GITTOP=`git rev-parse --show-toplevel`
 cd $GITTOP
 
 npm run ahoy
 [ $? -ne 0 ] && printf "\n\n***Aborting helmup...\n\n" && exit 1
 
-helm repo update && helm search chartmuseum
+helm repo update &> /dev/null && helm search chartmuseum
 
 function doInstall() {
 
@@ -41,6 +164,7 @@ function doInstall() {
     case "$SERVICE" in
       "platform")
         doInstall "${PLATFORM[@]}"
+        printf "\nkeel may be installed by running => helmup keel\n\n"
         ;;
 
       "system")
@@ -49,35 +173,83 @@ function doInstall() {
 
       "descheduler")
         helm repo add akomljen-charts     https://raw.githubusercontent.com/komljen/helm-charts/master/charts/
-        helm upgrade --install descheduler akomljen-charts/descheduler --set schedule="*/13 * * * *"
+        helm upgrade --install descheduler akomljen-charts/descheduler --set schedule="*/15 * * * *",deschedulerPolicy.strategies.RemovePodsViolatingNodeAffinity.params.nodeAffinityType={preferredDuringSchedulingIgnoredDuringExecution}
+        ;;
+
+      "disks")
+        checkSSDStorageClass
+        helm upgrade --install disks ./disks
         ;;
 
       "elastic"|"elasticsearch")
+        checkSSDStorageClass
         helm repo add elastic https://helm.elastic.co
-        helm upgrade --install elasticsearch elastic/elasticsearch --version 7.2.0
+        helm upgrade --install elasticsearch elastic/elasticsearch --set volumeClaimTemplate.storageClassName="ssd"
+        ;;
+
+      "keel")
+        installKeel # because it's a little more involved
         ;;
 
       "monitoring")
         helm dep up ./monitoring && helm upgrade --install monitoring ./monitoring
         ;;
 
+      "polaris")
+        helm repo add reactiveops-stable https://charts.reactiveops.com/stable
+        helm upgrade --install polaris reactiveops-stable/polaris --namespace polaris
+        printf "\n***Proxy 8081 with this=> kubectl port-forward --namespace polaris svc/polaris-dashboard 8081:80\n\n"
+        ;;
+
       "redis")
         # https://github.com/helm/charts/tree/master/stable/redis
         helm upgrade --install redis stable/redis \
           --set usePassword=false,cluster.slaveCount=3,networkPolicy.enabled=true,metrics.enabled=true
         ;;
 
+      "timescale-db")
+        helm dep up ./timescale-db && helm upgrade --install timescale-db ./timescale-db
+        ;;
+
       "ingress")
         helm upgrade --install ingress ./ingress
         ;;
 
       *)
-        [ ! -d $SERVICE ] && continue
-        [ ! -f $SERVICE/values-local.yaml ] && continue
+        if [ ! -d $SERVICE ] || [ ! -f $SERVICE/values-local.yaml ];
+        then
+          printf "\n$YELO_WARN Skipping unknown service => $SERVICE\n\n"
+          continue
+        fi
 
         printf "\n--------------------------------------------------------------\nCharting $SERVICE\n"
-        helm upgrade --install $SERVICE chartmuseum/$SERVICE -f $SERVICE/values.yaml
-        [ $? -ne 0 ] && printf "The $SERVICE chart failed - you have issues.\n"
+        # By default the museum chart name will be the same as the service, but
+        # we have the flexibility to override the naming when the charts are
+        # reference by the downstream repositories via the .helmup-charmuseum file.
+        OVERRIDE="$SERVICE/.helmup-chartmuseum"
+        MUSEUM=$SERVICE
+        if [ -f $OVERRIDE ];
+        then
+          MUSEUM=$(<${OVERRIDE})
+          printf "\n$YELO_WARN Local chartmuseum name override from ${SERVICE} to ${MUSEUM}\n"
+        fi
+        helm upgrade --install $SERVICE chartmuseum/$MUSEUM -f $SERVICE/values.yaml 2> /dev/null
+        if [ $? -ne 0 ];
+        then
+          cat<<NOTRUNNING
+
+$RED_ERROR The $SERVICE chart failed to upgrade or install. Is chartmuseum up 
+           and running on the proper port? Try firing it up with:
+
+chartmuseum \\
+  --port="8080" \\
+  --storage-google-bucket="leverege-helm-charts" \\
+  --storage-google-prefix="" \\
+  --storage="google" &> /dev/null &
+
+NOTRUNNING
+          exit 1
+        fi
         ;;
     esac
   done

package/src/overwhelm

@@ -11,16 +11,19 @@ const ask = require( 'readline-sync' )
 const YAML = require( 'js-yaml' )
 
 const optionsDefinitions = [
+  { name : 'context',  alias : 'x', type : String, default : '' },
   { name : 'section',  alias : 's', type : String, default : 'default' },
   { name : 'key',      alias : 'k', type : String },
   { name : 'export',   alias : 'e', type : Boolean, default : false },
   { name : 'continue', alias : 'c', type : Boolean, default : false },
+  { name : 'checkctx', alias : 't', type : Boolean, default : false },
   { name : 'verbose',  alias : 'v', type : Boolean, default : false },
 ]
 const args = cliArgs( optionsDefinitions )
 
 // The kubernetes context is equal to the git branch name.
-const context = require( 'git-branch' ).sync()
+const gitBranch = require( 'git-branch' ).sync()
+const context   = args.context || gitBranch
 
 if ( context == 'master' ) {
   console.log( '***ERROR: master context is forbidden; change to a different branch\n\n' )
@@ -36,15 +39,17 @@ if ( context == 'master' ) {
 //   my-project:
 //     GCE_MACHINE : "n1-highcpu-8"
 //
-// where the my-project specific setting may override the default if it
-// needs a beefier deployment. Any setting can be defaulted in this
 // fashion, but it is advised not to set a default region with this
 // approach in order to keep us cognizant of exactly where a cluster is
 // being formed.
 //
 const yaml = YAML.safeLoad( fs.readFileSync( 'overwhelm.yaml', 'utf-8' ) )
 const over = yaml.default
-const cfgs = yaml[context]
+let   cfgs = yaml[context]
+let   wtfs = cfgs
+
+// If we just need to check context return 0 if cfgs exists, else error.
+if ( args.checkctx ) { process.exit( !cfgs ) }
 
 if ( !cfgs ) {
   console.log( `***ERROR: Unknown context => [${context}]\n` )
@@ -56,6 +61,17 @@ if ( !cfgs ) {
   process.exit( 1 )
 }
 
+// Check to see if we are deriving from another configuration.
+if ( cfgs['PROJECT_BASE'] ) {
+  const baseName = cfgs['PROJECT_BASE']
+  const tempCfgs = cfgs
+  cfgs = yaml[baseName]
+  Object.keys( tempCfgs ).forEach( ( key ) => { cfgs[key] = tempCfgs[key] } )
+  console.log( "PROJECT_BASE still under evaluation" )
+  console.log( "\n  cfgs=", cfgs )
+  process.exit( 1 ) //XXX HERE
+}
+
 Object.keys( cfgs ).forEach( ( key ) => { over[key] = cfgs[key] || over[key] } )
 
 // Expand any variables that reference other variable settings. A perfect
@@ -232,7 +248,7 @@ ahoy.forEach( ( dir ) => {
 
 // Set the GCP context for safety!
   const clusterName = over['CLUSTER_NAME']
-  const projectName = over['PROJECT_NAME']
+  const projectId   = over['PROJECT_ID']
   const zone = over['GCE_REGION'] + ( over['GCE_ZONE'] ? `-${over['GCE_ZONE']}` : '' )
 
   if ( !clusterName ) {
@@ -248,7 +264,7 @@ ahoy.forEach( ( dir ) => {
       ${chalk.bold.green( `CLUSTER_NAME   : "\${PROJECT_NAME}"` )}
 
     However, if the CLUSTER_NAME is not the same as the PROJECT_NAME then go
-    ahead and add the CLUSTER_NAME setting to the ${chalk.bold.green(projectName)} section of the
+    ahead and add the CLUSTER_NAME setting to the ${chalk.bold.green(projectId)} section of the
     overwhelm.yaml file.
 
     ` ) )
@@ -262,9 +278,9 @@ ahoy.forEach( ( dir ) => {
   } catch (err) { }
 
   const currentCtx = kubeout.toString().trim()
-  const desiredCtx = `gke_${projectName}_${zone}_${clusterName}`
+  const desiredCtx = `gke_${projectId}_${zone}_${clusterName}`
   const matchedCtx = ( currentCtx === desiredCtx )
-  const cmd = `gcloud container clusters get-credentials ${clusterName} --zone ${zone} --project ${projectName}`
+  const cmd = `gcloud container clusters get-credentials ${clusterName} --zone ${zone} --project ${projectId}`
   
   if ( matchedCtx ) {
     console.log( `
@@ -278,7 +294,7 @@ ahoy.forEach( ( dir ) => {
 
   You are about to change your GCP context to:
       CLUSTER_NAME => ${chalk.bold.green(clusterName)}
-      PROJECT_NAME => ${chalk.bold.green(projectName)}
+      PROJECT_ID   => ${chalk.bold.green(projectId)}
       GCE_REGION   => ${chalk.bold.green(zone)}
 
   ` )

package/src/pkglint/index.js

@@ -16,6 +16,7 @@ function arrayContainsArray( superset, subset ) {
   } )
 }
 
-arrayContainsArray( mdFiles, filesToCheck ) ? 
-  console.log( 'README and LICENSE files exist' ) :
+if ( !arrayContainsArray( mdFiles, filesToCheck ) ) {
+  console.log( 'Cannot find README.md or LICENSE.md' )
   process.exit( 1 )
+}  

package/lib/server/auditate/help.js

@@ -1,119 +0,0 @@
-"use strict";
-
-const groupBy = require('lodash.groupby');
-
-const keyBy = require('lodash.keyby');
-
-const countBy = require('lodash.countby');
-
-const orderBy = require('lodash.orderby');
-
-const map = require('lodash.map');
-
-const first = require('lodash.first');
-
-module.exports = (config = {}) => {
-  function affectsProd(resolves) {
-    return !(config['prod-only'] && resolves.every(({
-      dev
-    }) => dev));
-  }
-
-  function requireAttention(auditResult) {
-    const {
-      actions,
-      advisories
-    } = auditResult;
-    const reviewActions = actions.filter(({
-      action,
-      resolves,
-      isMajor
-    }) => (action === 'review' || isMajor) && affectsProd(resolves));
-    const reviewAdvisories = reviewActions.reduce((acc, {
-      resolves
-    }) => {
-      const advisoriesToInclude = resolves.reduce((result, {
-        id
-      }) => Object.assign({}, result, {
-        [id]: advisories[id]
-      }), {});
-      return Object.assign({}, acc, advisoriesToInclude);
-    }, {});
-    return Object.assign({}, auditResult, {
-      actions: reviewActions,
-      advisories: reviewAdvisories
-    });
-  }
-
-  function getHighestSeverity({
-    advisories
-  }) {
-    for (const severity of ['critical', 'high', 'moderate', 'low']) {
-      if (Object.keys(advisories).find(key => advisories[key].severity === severity)) {
-        return severity;
-      }
-    }
-  }
-
-  function filterAuditResultsBySeverity(auditResult, highestSeverity) {
-    const {
-      actions,
-      advisories
-    } = auditResult;
-    const groupedAdvisories = groupBy(advisories, 'severity');
-    const advisoriesForSeverity = keyBy(groupedAdvisories[highestSeverity], 'id');
-    const actionsForSeverity = actions.reduce((acc, action) => {
-      const resolvesSevereAdvisory = action.resolves.some(({
-        id
-      }) => advisoriesForSeverity[id]);
-      return resolvesSevereAdvisory ? [].concat(acc, action) : acc;
-    }, []);
-    return Object.assign({}, auditResult, {
-      actions: actionsForSeverity,
-      advisories
-    });
-  }
-
-  function getAutoFixCount(auditResult) {
-    return auditResult.actions.filter(({
-      action,
-      resolves,
-      isMajor
-    }) => action !== 'review' && !isMajor && affectsProd(resolves)).length;
-  }
-
-  function findRootLibs(resolves) {
-    return resolves.reduce((acc, resolve) => [].concat(acc, resolve.path.split('>')[0]), []);
-  }
-
-  function getMostProblematicDependency({
-    actions
-  }) {
-    const rootLibraries = actions.reduce((libs, {
-      resolves
-    }) => [].concat(libs, findRootLibs(resolves)), []);
-    return first(orderBy(map(countBy(rootLibraries), (count, lib) => ({
-      count,
-      name: lib
-    })), ({
-      count
-    }) => count, ['desc']));
-  }
-
-  return function help(npmAuditResultJson) {
-    const npmAuditResult = JSON.parse(npmAuditResultJson);
-    const resultsThatRequireAttention = requireAttention(npmAuditResult);
-    const highestSeverity = getHighestSeverity(resultsThatRequireAttention);
-    const auditResult = filterAuditResultsBySeverity(resultsThatRequireAttention, highestSeverity);
-    const autoFixCount = getAutoFixCount(npmAuditResult);
-    const exitCode = config['exit-zero'] || !auditResult.actions.length && !autoFixCount ? 0 : 1;
-    return {
-      npmAuditResult,
-      auditResult,
-      autoFixCount,
-      mostProblematicDependency: getMostProblematicDependency(resultsThatRequireAttention),
-      highestSeverity,
-      exitCode
-    };
-  };
-};

package/lib/server/auditate/index.js

@@ -1,23 +0,0 @@
-#!/usr/bin/env node
-"use strict";
-
-const getStdin = require('get-stdin');
-
-const options = require('minimist')(process.argv.slice(2));
-
-const help = require('./help');
-
-const report = require('./report');
-
-const defaults = {
-  'exit-zero': false,
-  'prod-only': false
-};
-const config = Object.assign({}, defaults, options);
-getStdin().then(help(config)).then(result => report(result).then(theReport => {
-  console.log(theReport);
-  process.exit(result.exitCode);
-})).catch(error => {
-  console.error(error);
-  process.exit(1);
-});

package/lib/server/auditate/report.js

@@ -1,61 +0,0 @@
-"use strict";
-
-const auditReport = require('npm-audit-report');
-
-const chalk = require('chalk');
-
-function automaticFixes(autoFixCount) {
-  return autoFixCount ? `run ${chalk.yellow('`npm audit fix`')} to automatically fix ${chalk.yellow(autoFixCount)} issues. These should all be non-breaking upgrades, so don't stress.` : '';
-}
-
-function fixesToReview({
-  actions,
-  advisories
-}, highestSeverity) {
-  const advisoryIds = Object.keys(advisories);
-
-  if (!actions.length || !advisoryIds.length) {
-    return '';
-  }
-
-  return `Resolve the ${chalk.green(actions.length)} ${chalk.green(highestSeverity)} severity issues above and run this command again to move to the next severity.`;
-}
-
-function mostProblematic(mostProblematicDependency) {
-  if (!mostProblematicDependency) {
-    return '';
-  }
-
-  const {
-    name,
-    count
-  } = mostProblematicDependency;
-  return count && count > 2 ? `The most problematic dependency seems to be ${chalk.blue(name)} with ${chalk.blue(count)} issue${count ? 's' : ''} that need your attention.` : '';
-}
-
-module.exports = async function printFullReport({
-  npmAuditResult,
-  auditResult,
-  autoFixCount,
-  mostProblematicDependency,
-  highestSeverity
-}) {
-  const {
-    report
-  } = await auditReport(npmAuditResult, {
-    reporter: 'install',
-    withColor: true,
-    withUnicode: true
-  });
-  const hints = [].concat(automaticFixes(autoFixCount)).concat(fixesToReview(auditResult, highestSeverity)).concat(mostProblematic(mostProblematicDependency)).filter(Boolean);
-  const help = hints.length ? `
-
-${chalk.green('=== A little bit of help ===')}
-
-Where to start:
-${hints.reduce((acc, hint) => `${acc}
-- ${hint}
-`, '')}
-` : '';
-  return `${report}${help}`;
-};