@leverege/build-tools 2.2.1 → 2.2.3

package/CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [2.2.3] = 2020-04-22
+### Updated
+- (Helmify) Added a CLI interface for Helmup
+
 ## [2.1.5] - 2020-03-08
 ### Updated
 - (Helmup) Simplified Timescale installation process

package/lib/server/helmify/files.js

@@ -0,0 +1,24 @@
+"use strict";
+
+const fs = require('fs');
+
+const path = require('path');
+
+const inquirer = require('inquirer');
+
+module.exports = {
+  getCurrentDirectoryBase: () => {
+    return path.basename(process.cwd());
+  },
+  directoryExists: filePath => {
+    return fs.existsSync(filePath);
+  },
+  overwriteDirectory: dirName => {
+    const question = [{
+      name: 'overwriteDirectory',
+      type: 'confirm',
+      message: `Overwrite existing ${dirName} directory?`
+    }];
+    return inquirer.prompt(question);
+  }
+};

package/lib/server/helmify/helm.js

@@ -0,0 +1,219 @@
+"use strict";
+
+const fs = require('fs');
+
+const fsp = fs.promises;
+
+const path = require('path');
+
+const inquirer = require('inquirer');
+
+const chalk = require('chalk');
+
+const execa = require('execa');
+
+const yaml = require('js-yaml');
+
+const packageJsonPath = path.join(process.cwd(), 'package.json');
+
+const packageJson = require(`${packageJsonPath}`);
+
+function sanitizePackageName(name) {
+  let validName = name; // Remove npm lib name (e.g. @leverege)
+
+  const isNpm = name.startsWith('@');
+
+  if (isNpm) {
+    const splitStr = name.split('/');
+    validName = splitStr[1]; // returns name after npm domain
+  }
+
+  return validName;
+}
+
+function transformGoFormat(input) {
+  return input.replace(/-/g, '_');
+}
+
+async function createChartYAML(chart) {
+  try {
+    await fsp.writeFile('helm/Chart.yaml', yaml.safeDump(chart), 'utf8');
+  } catch (err) {
+    console.error('Failed to create Chart.yaml file', chalk.red.bold('ERROR'));
+    process.exit(1);
+  }
+}
+
+async function createValuesYAML(name, version, redisAccess, securityContext, preemptible, service) {
+  const openPort = service !== 'none';
+  const portFromEnv = process.env.PORT || process.env.SERVER_PORT;
+  const portNumber = parseInt(portFromEnv) || 8080;
+  const values = {
+    [name]: {
+      "VERSION": version,
+      "PREMPTIBLE": preemptible,
+      ...(openPort && {
+        port: portNumber
+      })
+    },
+    "redisClient": {
+      "label": redisAccess
+    },
+    "securityContext": {
+      "readOnlyRootFs": !securityContext
+    },
+    "cloud": {
+      "provider": "gcp",
+      "project": "",
+      "region": "",
+      "sqlInstance": ""
+    }
+  };
+
+  try {
+    await fsp.writeFile('helm/values.yaml', yaml.safeDump(values), 'utf8');
+  } catch (err) {
+    console.error('Failed to create values.yaml file', chalk.red.bold('ERROR'));
+    process.exit(1);
+  }
+}
+
+async function createServiceYAML(name, serviceType) {
+  try {
+    await execa('helminit-svc', [name, serviceType]);
+  } catch (err) {
+    console.error('Failed to create service.yaml file', chalk.red.bold('ERROR'));
+    process.exit(1);
+  }
+}
+
+async function createDeploymentYAML(name, enableFirebase, enableBigQuery) {
+  try {
+    await execa('helminit-dp', [name, enableFirebase, enableBigQuery]);
+  } catch (err) {
+    console.error('Failed to create deployment.yaml file', chalk.red.bold('ERROR'));
+    process.exit(1);
+  }
+}
+
+async function createScalerYAML(name) {
+  try {
+    await execa('helminit-scaler', [name]);
+  } catch (err) {
+    console.error('Failed to create scaler.yaml file', chalk.red.bold('ERROR'));
+    process.exit(1);
+  }
+}
+
+async function createTemplates() {
+  const helmQuestions = [{
+    name: 'redisAccess',
+    type: 'confirm',
+    message: 'Does this chart need access to Redis (e.g. cache, limit, lock options)?'
+  }, {
+    name: 'preemptible',
+    type: 'confirm',
+    message: 'Is this server stateless and can survive occasional unavailability?'
+  }, {
+    name: 'securityContext',
+    type: 'confirm',
+    message: 'Does this server need special write access to filesystem?'
+  }, {
+    name: 'service',
+    type: 'rawlist',
+    message: 'Does this server need a TCP port open for internal or external access?',
+    choices: ['none', 'internal', 'external'],
+    default: ['none']
+  }, {
+    name: 'scaler',
+    type: 'confirm',
+    message: 'Does this server need horizontal scaling?'
+  }, {
+    name: 'cloud',
+    type: 'rawlist',
+    message: 'By default, which cloud platform will this server be deployed on?',
+    choices: ['GCP', 'AWS'],
+    default: ['GCP']
+  }, {
+    name: 'cloudServices',
+    type: 'checkbox',
+    message: 'Which these cloud services does this server need? (space to choose, enter to submit)',
+    choices: ['firebase', 'bigquery']
+  }];
+  const helmOptions = await inquirer.prompt(helmQuestions);
+  return helmOptions;
+}
+
+module.exports = {
+  bootstrap: async () => {
+    let name = sanitizePackageName(packageJson.name);
+    const appVersion = packageJson.version;
+    const keepNameQuestion = [{
+      name: 'keepPackageName',
+      type: 'confirm',
+      message: `Helm chart name will default to ${name}. Do you want to keep the default setting?`
+    }];
+    const keepName = await inquirer.prompt(keepNameQuestion);
+
+    if (!keepName.keepPackageName) {
+      const keepNameQuestion = [{
+        name: 'renamePackage',
+        type: 'input',
+        message: 'Please enter a new name for the Helm chart',
+        validate: str => {
+          const helmRegex = RegExp('^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])+$');
+
+          if (helmRegex.test(str)) {
+            return true;
+          } else {
+            return 'Special characters are not allowed. Please enter a valid name.';
+          }
+        }
+      }];
+      const newName = await inquirer.prompt(keepNameQuestion);
+      name = newName.renamePackage;
+    }
+
+    console.log(chalk.green(`\nBootstrapping a new Helm chart for ${name}...\n`));
+    const chart = {
+      name,
+      version: "1.0.0",
+      appVersion
+    };
+    const chartSafeName = transformGoFormat(name);
+
+    try {
+      await execa('mkdir', ['helm']);
+      await execa('mkdir', ['helm/templates']);
+    } catch (err) {
+      console.error('Failed to create directories', chalk.red.bold('ERROR'));
+      process.exit(1);
+    }
+
+    const templateOpts = await createTemplates();
+    const {
+      service,
+      scaler,
+      cloudServices,
+      redisAccess,
+      securityContext,
+      preemptible
+    } = templateOpts;
+
+    if (service !== 'none') {
+      const serviceType = service === 'internal' ? 'ClusterIP' : 'NodePort';
+      await createServiceYAML(chartSafeName, serviceType);
+    }
+
+    if (scaler) {
+      await createScalerYAML(chartSafeName);
+    }
+
+    const enableFirebase = cloudServices.includes('firebase');
+    const enableBigQuery = cloudServices.includes('bigquery');
+    await createDeploymentYAML(name, enableFirebase, enableBigQuery);
+    await createValuesYAML(chartSafeName, appVersion, redisAccess, securityContext, preemptible, service);
+    console.log(chalk.green.bold(`\n\n Helm chart for ${name} is now ready \n`));
+    console.log(chalk.yellow.bold('Add any additional env vars to value.yaml and deployment.yaml prior to deployment'));
+  }
+};

package/lib/server/helmify/index.js

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+"use strict";
+
+require('dotenv').config(); // attempt to autodetect envs
+
+
+const chalk = require('chalk');
+
+const clear = require('clear');
+
+const figlet = require('figlet');
+
+const execa = require('execa');
+
+const files = require('./files');
+
+const helm = require('./helm');
+
+clear();
+console.log(chalk.yellow(figlet.textSync('Helmify', {
+  horizontalLayout: 'full'
+})));
+
+const run = async () => {
+  if (files.directoryExists('helm')) {
+    console.log(chalk.red.bold('\nDetected existing Helm configs\n'));
+    const overwrite = await files.overwriteDirectory('Helm');
+
+    if (!overwrite.overwriteDirectory) {
+      console.log(chalk.green('\nNo changes made - exiting... \n'));
+      process.exit();
+    }
+
+    console.log(chalk.yellow.bold('\nDeleting existing Helm directory...\n'));
+    const deleteHelmDir = await execa('rm', ['-rf', 'helm']);
+
+    if (deleteHelmDir.failed) {
+      console.log(chalk.red.bold('\nFailed to delete existing Helm directory\n'));
+      process.exit(1);
+    }
+  }
+
+  helm.bootstrap();
+};
+
+run();

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.2.1",
+  "version": "2.2.3",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "scripts": {
     "build": "npm run compile",
     "compile": "rm -fr lib && npm run compile_server && npm run compile_web",
-    "compile_server": "NODE_ENV=server babel -d lib/server src/ && cp src/dnsify/accounts.json.enc lib/server/dnsify/accounts.json.enc",
+    "compile_server": "NODE_ENV=server babel -d lib/server src/",
     "compile_web": "NODE_ENV=web babel -d lib/web src/",
     "licenseCheck": "license-checker --summary --failOn GPL",
     "prepublintOnly": "npm run lint && npm run build",
@@ -17,14 +17,16 @@
     "circleate": "src/circleate",
     "decrypt-secrets": "src/decrypt-secrets",
     "dirty-git": "src/dirty-git",
-    "dnsify": "lib/server/dnsify/index.js",
     "dockreate": "src/dockreate",
     "encrypt-secrets": "src/encrypt-secrets",
     "getfbcfg": "src/getfbcfg",
     "getjson": "src/getjson",
     "git-tar": "src/git-tar",
     "helmdn": "src/helmdn",
-    "helmify": "src/helmify",
+    "helmify": "lib/server/helmify/index.js",
+    "helminit-svc": "src/helminit-svc",
+    "helminit-dp": "src/helminit-dp",
+    "helminit-scaler": "src/helminit-scaler",
     "helmup": "src/helmup",
     "k8x": "src/k8x",
     "overwhelm": "src/overwhelm",
@@ -40,14 +42,17 @@
     "@google-cloud/kms": "^1.6.3",
     "async": "^3.2.0",
     "chalk": "^3.0.0",
-    "cloudflare": "^2.7.0",
-    "command-line-args": "^5.1.1",
-    "command-line-usage": "^5.0.5",
+    "clear": "^0.1.0",
     "commander": "^2.20.3",
+    "dotenv": "^8.2.0",
+    "execa": "^4.0.0",
+    "figlet": "^1.3.0",
     "fs-extra": "^8.1.0",
     "git-branch": "^2.0.1",
     "glob": "^7.1.6",
+    "inquirer": "^7.1.0",
     "js-yaml": "^3.13.1",
+    "minimist": "^1.2.5",
     "ora": "^3.4.0",
     "parse-gitignore": "^1.0.1",
     "readline-sync": "^1.4.10",

package/src/circleate

@@ -93,7 +93,7 @@ jobs:
   smoke-test:
     # smoke test checks for license, security, and linting
     docker: 
-      - image: circleci/node:12.10.0
+      - image: circleci/node:erbium
     steps:
       - checkout
       - restore_cache:
@@ -110,7 +110,7 @@ jobs:
 
       - run: 
           name: Check for vulnerabilities
-          command: npm audit --production
+          command: npm audit --production --audit-level=moderate
 
       - run:
           name: Check for LICENSE and README files
@@ -134,7 +134,7 @@ jobs:
     environment:
       TZ: 'America/New_York'
     docker:
-      - image: circleci/node:12.10.0
+      - image: circleci/node:erbium
       # IF YOU NEED ANY OF THESE, UNCOMMENT TO USE
       # - image: redis
       # - image: circleci/mysql:5.7
@@ -195,7 +195,7 @@ jobs:
   build:
     # build should make dist image and prepare docker file
     docker:
-      - image: circleci/node:12.10.0
+      - image: circleci/node:erbium
 
     steps:
       - checkout
@@ -281,7 +281,7 @@ jobs:
   smoke-test:
     # smoke test checks for license, security, and linting
     docker: 
-      - image: circleci/node:12.10.0
+      - image: circleci/node:erbium
     steps:
       - checkout
       - restore_cache:
@@ -298,7 +298,7 @@ jobs:
 
       - run: 
           name: Check for vulnerabilities
-          command: npm audit --production
+          command: npm audit --production --audit-level=moderate
 
       - run:
           name: Check for LICENSE and README files
@@ -320,7 +320,7 @@ jobs:
   coverage:
     # coverage runs tests, generates coverage, and saves docs
     docker:
-      - image: circleci/node:12.10.0
+      - image: circleci/node:erbium
       # IF YOU NEED ANY OF THESE, UNCOMMENT TO USE
       # - image: redis
       # - image: circleci/mysql:5.7
@@ -473,7 +473,7 @@ jobs:
 
   build:
     docker:
-      - image: circleci/node:12.10.0
+      - image: circleci/node:erbium
     steps:
       - checkout
       - restore_cache:

package/src/helminit-dp

@@ -0,0 +1,174 @@
+#!/bin/bash
+
+dockerName=$1
+chartName=${dockerName//-/_}
+
+enableFirebase=$2
+enableBigQuery=$3
+
+if [ "$enableFirebase" = true ] ; then
+  firebaseVolume="
+      - name: rt-firebase-key
+        secret:
+          secretName: rt-data
+      - name: model-firebase-key
+        secret:
+          secretName: model-data
+      - name: model-firebase-config
+        secret:
+          secretName: model-data-config
+      - name: rt-firebase-config
+        secret:
+          secretName: rt-data-config"
+
+  firebaseVolumeMount='
+        - name: model-firebase-key
+          mountPath: "/home/node/.gcp/model-data"
+        - name: rt-firebase-key
+          mountPath: "/home/node/.gcp/rt-data"
+        - name: model-firebase-config
+          mountPath: "/home/node/.gcp/model-data-config"
+        - name: rt-firebase-config
+          mountPath: "/home/node/.gcp/rt-data-config"'
+fi
+
+if [ "$enableBigQuery" = true ] ; then
+  bigQueryVolume="
+      - name: big-query
+        secret:
+          secretName: big-query"
+
+  bigQueryVolumeMount='
+        - name: big-query
+          mountPath: "/home/node/.gcp/bigquery"'
+fi
+
+touch helm/templates/deployment.yaml
+
+cat<<EODEPLOYMENT > helm/templates/deployment.yaml
+{{- if .Values.cloud.provider -}}
+{{- \$provider := .Values.cloud.provider -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}
+  labels:
+    app: {{ .Chart.Name }}
+    chart/version: {{ .Chart.Version }}
+    chart/appVersion: {{ .Chart.AppVersion }}
+    redis-client: {{ .Values.redisClient.label | quote }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Chart.Name }}
+        chart/version: {{ .Chart.Version }}
+        chart/appVersion: {{ .Chart.AppVersion }} 
+        redis-client: {{ .Values.redisClient.label | quote }}
+      annotations:
+        prometheus.io/port: '5111'
+        prometheus.io/scrape: 'true'
+        configmap.reloader.stakater.com/reload: "default-configs"
+    spec:
+      volumes: $firebaseVolume $bigQueryVolume
+      {{- if eq \$provider "gcp" }}
+      - name: pubsub-key
+        secret:
+          secretName: pubsub
+      {{- else if eq \$provider "aws" }} 
+      - name: sns-sqs-key
+        secret:
+          secretName: sns-sqs
+      {{- end }}
+      - name: tmpfs
+        emptyDir: {}
+      {{- if .Values.$chartName.PREEMPTIBLE }}
+      tolerations:
+      - key: "preemptible"
+        operator: "Equal"
+        value: "true"
+        effect: "NoSchedule"
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: cloud.google.com/gke-preemptible
+                operator: Exists
+            weight: 100
+      {{- end }}
+      containers:
+      - name: {{ .Chart.Name }}
+        imagePullPolicy: Always
+        image: gcr.io/leverege-docker-images/$dockerName:{{ .Values.$chartName.VERSION }}
+        securityContext:
+          readOnlyRootFilesystem: {{ .Values.securityContext.readOnlyRootFs }}
+          runAsNonRoot: true
+          runAsUser: 1000
+          allowPrivilegeEscalation: false
+        # UNCOMMENT IF YOU NEED PROBES
+        # readinessProbe:
+        #   httpGet:
+        #     path: /healthz
+        #     port: api
+        #   initialDelaySeconds: 10
+        # livenessProbe:
+        #   httpGet:
+        #     path: /livenessProbe
+        #     port: api
+        #   initialDelaySeconds: 15
+        #   periodSeconds: 60
+        ports:
+        {{- if .Values.$chartName.port }}
+        - name: api
+          containerPort: {{ .Values.$chartName.port }}
+        {{- end }}
+        - name: node-metrics
+          containerPort: 5111
+        resources:
+          requests:
+            cpu: "50m"
+            memory: "128Mi"
+          limits:
+            cpu: "100m"
+            memory: "256Mi"
+        volumeMounts: $firebaseVolumeMount $bigQueryVolumeMount
+        {{- if eq \$provider "gcp" }}
+        - name: pubsub-key
+          mountPath: "/home/node/.gcp/pubsub"
+        {{- else if eq \$provider "aws" }}        
+        - name: sns-sqs-key
+          mountPath: "/home/node/.aws"
+        {{- end }}
+        - name: tmpfs
+          mountPath: "/tmp"
+        envFrom: 
+        - configMapRef:
+            name: default-configs
+        env:
+        # UNCOMMENT THE FOLLOWING FOR ENV VARIABLES
+        # Assuming you have the following lines in values.yaml =>
+        #
+        # $chartName:
+        #   MY_ENV_VARIABLE: hello-world 
+        #
+        #
+        # - name: MY_ENV_VARIABLE
+        #   value: {{ .Values.$chartName.MY_ENV_VARIABLE }}
+        #
+        #
+        # UNCOMMENT THE FOLLOWING FOR MOUNTING SECRETS
+        # Assuming you ran => 
+        #   kubectl create secret generic my-secret --from-literal=my-password=password123
+        #
+        # - name: ENV_SECRET_NAME
+        #   valueFrom:
+        #     secretKeyRef:
+        #       name: my-secret
+        #       key: my-password
+
+{{- end -}}
+EODEPLOYMENT

package/src/helminit-scaler

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+chartName=$1
+
+touch helm/templates/scaler.yaml
+
+cat<<EOSCALER > helm/templates/scaler.yaml
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ .Chart.Name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Chart.Name }}
+  minReplicas: {{ .Values.horizontalScaler.minReplicas }}
+  maxReplicas: {{ .Values.horizontalScaler.maxReplicas }}
+  targetCPUUtilizationPercentage: {{ .Values.horizontalScaler.maxCPU }}
+EOSCALER

package/src/helminit-svc

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+chartName=$1
+serviceType=$2
+
+touch helm/templates/service.yaml
+
+cat<<EOSERVICE > helm/templates/service.yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: {{ .Chart.Name }}
+spec:
+  type: $serviceType
+  selector:
+    app: {{ .Chart.Name }}
+  ports:
+  - protocol: TCP
+    port: {{ .Values.$chartName.serverPort }}
+    targetPort: api
+EOSERVICE

package/src/helmup

@@ -458,7 +458,7 @@ function doInstall() {
         ;;
 
       "reloader")
-        helm upgrade --install reloader stable/reloader
+        helm upgrade --install reloader stakater/reloader
         ;;
 
       *)