@leverege/build-tools 2.38.2 → 2.39.0-beta.2

package/src/helm-charts/firebase-rules/leverege.rules.json

@@ -0,0 +1,65 @@
+{
+  "rules": {
+    "projectRefs": {
+      "$project_id": {
+        "member": {
+          ".indexOn": ["created"]
+        }
+      }
+    },
+    "blueprint": {
+      ".indexOn": "lastModified"
+    },
+    "blueprintChildren": {
+      "$bp_id": {
+        "attributes": {
+          ".indexOn": ".value"
+        }
+      }
+    },
+    "attribute":{
+      ".indexOn" : [ "path", "blueprintId" ]
+    },
+    "systemChildren": {
+      "$sys_id": {
+        "device": {
+          ".indexOn": ".value"
+        }
+      }
+    },
+    "device": {
+      ".indexOn": ["id", "networkId", "simNetworkId"]
+    },
+    "deviceRefs": {
+      "$dev_id": {
+        "member": {
+          ".indexOn": ["created"]
+        }
+      }
+    },
+    "user": {
+      ".indexOn": ["username", "email"]
+    },
+    "project": {
+      ".indexOn": ["id"]
+    },
+    "rt": {
+      "$sys_id": {
+        ".read": "auth.systemId === $sys_id",
+        "dev": {
+          "$dev_id": {
+            ".read": "auth.deviceId === $dev_id",
+            "outbound": {
+              ".indexOn": ["initiated", "sent", "ack"]
+            },
+            "events": {
+              "$evt_type": {
+                ".indexOn": ["time", "id"]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/src/helm-charts/geotile-server/gitignore

@@ -0,0 +1 @@
+geotile-local.yaml

package/src/helm-charts/geotile-server/helmup.plugin

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+showInstalling "Geotile Server"
+
+helm upgrade --install geotile-server leverege/geotile-server \
+             --values geotile-server/geotile-values.yaml

package/src/helm-charts/velero/helmup.plugin

@@ -31,6 +31,22 @@ helm upgrade \
      --set configuration.backupStorageLocation[0].config.serviceAccount="$GCE_SA_EMAIL" \
      --set configuration.volumeSnapshotLocation[0].config.snapshotLocation="$GCE_REGION"
 
+if [ $? -ne 0 ];
+then
+  cat<<VELERROR
+
+  `color r ***ERROR:` helm upgrade has failed! If upgrading from an older chart there
+  may be residual velero components that need to be cleaned up. Try running the
+  following kubectl commands and then rerun helmup:
+
+      kubectl delete backupstoragelocations.velero.io -n velero default
+      kubectl delete volumesnapshotlocations.velero.io -n velero default
+      helmup velero
+
+VELERROR
+  exit 1
+fi
+
 velero schedule get
 cat<<DONE_VELERO
 

package/src/helmup.sh

@@ -152,10 +152,29 @@ MISSING_HELMUP_PLUGIN
   # because npm tends to not allow .gitinore files the easy way...
   [ -f "$SERVICE/gitignore" ] && mv $SERVICE/gitignore $SERVICE/.gitignore
   git add $SERVICE
-  overwhelm.js --genvals
+  overwhelm --genvals
   ifPluggedIn $SERVICE
 }
 
+function installDefaultConfigs() {
+  showInstalling "Default Configs"
+
+  clearlyWarn 'without clearing'
+  cat<<INSTALLDEFAULTCONFIGS
+  Continuing with applying the default-configs.yaml will cause all non-system
+  services to `color r restart`. Make certain people are made aware that this is about
+  to happen.
+
+INSTALLDEFAULTCONFIGS
+  yesToContinue "to apply the default-configs to the cluster"
+
+  kubecmd="kubectl apply -f default-configs.yaml"
+
+  printf "\nRunning => `color y \"$kubecmd\"`\n\n"
+  eval $kubecmd
+  postToSlack "$kubecmd"
+}
+
 function errorIfMissingSecret() {
   SECRET="$1"
   kubectl get secret $SECRET &> $DEVNULL
@@ -165,6 +184,24 @@ function errorIfMissingSecret() {
   fi
 }
 
+function installFirebaseDatabases() {
+  showInstalling "Firebase Realtime Databases"
+
+  open https://console.firebase.google.com/u/0/project/${GCP_PROJECT_ID}/authentication/users
+  yesToContinue "when you have pressed Get Started on the Firebase Authentication page"
+
+  # create the firebase primary db
+  firebase --project $GCP_PROJECT_ID database:instances:create ${PROJECT}
+
+  # TODO: should maybe be more flexible with number of shards in the future
+  for SHARD in {0..9}
+  do
+    SHARD="${GCP_PROJECT_ID}-rt-${SHARD}"
+    printf "Creating shard $SHARD\n"
+    firebase --project $GCP_PROJECT_ID database:instances:create ${SHARD}
+  done
+}
+
 function installStackdriverExporterEnvironment() {
   showInstalling "Stackdriver Exporter Service Account Binding"
 
@@ -627,6 +664,10 @@ CATBACKUP
         kubectl create -f https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/production/adapter.yaml
         ;;
 
+      default-configs*)
+        installDefaultConfigs
+        ;;
+
       db-curator*)
         # db-curator / db-curator-dh diffs are defined in values-local.yaml
         HELMUP="helm upgrade --install $SERVICE leverege/db-curator --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
@@ -649,6 +690,11 @@ CATBACKUP
         installPreemptibleKiller # the gke preemptible node spreader outer
         ;;
 
+      "firebase")
+        installFirebaseDatabases
+        bootstrapLocalSetup firebase-rules
+        ;;
+
       "grafana")
         bootstrapLocalSetup $SERVICE
         ;;

package/src/overwhelm.mjs

@@ -1,22 +1,22 @@
 #!/usr/bin/env node
 
 // XXX console.log( `TOP=>[${process.cwd()}] PWD=[${process.env.PWD}]` )
-/* eslint-disable no-console */
 /* eslint-disable security/detect-non-literal-fs-filename */
 
 import path from 'node:path'
-import { execSync as exec } from 'node:child_process'
 import fs from 'node:fs'
 
 import chalk from 'chalk'
 import cliArgs from 'command-line-args'
 // const cliHelp = require( 'command-line-usage' )
 import deepmerge from 'deepmerge'
-import glob from 'glob'
+import { glob } from 'glob'
 import parse from 'parse-gitignore'
 import ask from 'readline-sync'
 import YAML from 'js-yaml'
 
+import { debug, log, shellCmd } from './Utils.mjs'
+
 const optionsDefinitions = [
   /* eslint-disable no-multi-spaces */
   { name : 'context',  alias : 'x', type : String, default : '' },
@@ -34,29 +34,26 @@ const optionsDefinitions = [
 const args = cliArgs( optionsDefinitions )
 
 // See if we are in a monorepo, which changes some of the rules.
-const gitTop = exec(
-  'git rev-parse --show-toplevel', { stdio : [ undefined ] }
-).toString().trim()
+const gitTop = await shellCmd( 'git rev-parse --show-toplevel' )
+
 let monoTop
 if ( fs.existsSync( `${gitTop}/packages` ) ) {
   monoTop = path.basename( path.resolve() ) // process.env.PWD
-  console.log( `MONO=[${monoTop}]` )
+  log( `MONO=[${monoTop}]` )
 }
 
 // The kubernetes context is equal to the git branch name.
-const gitBranch = exec(
-  'git rev-parse --abbrev-ref HEAD', { stdio : [ undefined ] }
-).toString().trim()
+const gitBranch = await shellCmd( 'git rev-parse --abbrev-ref HEAD' )
 
 const context = args.context || monoTop || gitBranch
 
 if ( context === 'master' ) {
-  console.log( '***ERROR: master context is forbidden; change to a different branch\n\n' )
+  log( '***ERROR: master context is forbidden; change to a different branch\n\n' )
   process.exit( 1 )
 }
 
 if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
-  console.log( chalk.red.bold( `
+  log( chalk.red.bold( `
 
 ***ERROR overwhelm => missing the overwhelm.yaml file - this command expects
                       to execute from within a platform-k8s branch
@@ -79,9 +76,8 @@ if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
 //
 const yaml = YAML.load( fs.readFileSync( 'overwhelm.yaml', 'utf-8' ) )
 const over = yaml.default
-if ( process.env.OVERWHELM_DEBUG ) {
-  console.log( { over, yaml }, '<==Contents of the over and yaml structs' )
-}
+
+debug( { over, yaml, }, '<==Contents of the over and yaml structs' )
 
 let cfgs = yaml[context]
 
@@ -89,12 +85,12 @@ let cfgs = yaml[context]
 if ( args.checkctx ) { process.exit( !cfgs ) }
 
 if ( !cfgs ) {
-  console.log( `***ERROR: Unknown context => [${context}]\n` )
-  console.log( '   Known k8s contexts are:' )
+  log( `***ERROR: Unknown context => [${context}]\n` )
+  log( '   Known k8s contexts are:' )
   Object.keys( yaml ).forEach( ( key ) => {
-    if ( key !== 'default' ) { console.log( `             ${key}` ) }
+    if ( key !== 'default' ) { log( `             ${key}` ) }
   } )
-  console.log( `Add a section into overwhelm.yaml for ${context}\n\n` )
+  log( `Add a section into overwhelm.yaml for ${context}\n\n` )
   process.exit( 1 )
 }
 
@@ -104,8 +100,8 @@ if ( cfgs.PROJECT_BASE ) {
   const tempCfgs = cfgs
   cfgs = yaml[baseName]
   Object.keys( tempCfgs ).forEach( ( key ) => { cfgs[key] = tempCfgs[key] } )
-  console.log( 'PROJECT_BASE still under evaluation' )
-  console.log( '\n  cfgs=', cfgs )
+  log( 'PROJECT_BASE still under evaluation' )
+  log( '\n  cfgs=', cfgs )
   process.exit( 1 ) // XXX HERE
 }
 
@@ -148,11 +144,11 @@ Object.keys( over ).forEach( ( key ) => {
 } )
 
 if ( varErrors.length > 0 ) {
-  console.log( '***ERROR: unkown variable reference(s) in overwhelm.yaml:\n' )
+  log( '***ERROR: unkown variable reference(s) in overwhelm.yaml:\n' )
   varErrors.forEach( ( error ) => {
-    console.log( `     ${error}` )
+    log( `     ${error}` )
   } )
-  console.log( '' )
+  log( '' )
   process.exit( 1 )
 }
 
@@ -170,7 +166,7 @@ const doReplacements = ( yamls, replacements ) => {
   }
 
   Object.keys( replacements ).forEach( ( repl ) => {
-    const replaceThis = new RegExp( `OVH:<${repl}>`, 'g' )
+    const replaceThis = new RegExp( `OVH:<${repl}>`, 'g' ) // eslint-disable-line security/detect-non-literal-regexp
     const withThat = quoteSpecials( `${replacements[repl]}` )
     replYaml = replYaml.replace( replaceThis, withThat )
   } )
@@ -179,11 +175,11 @@ const doReplacements = ( yamls, replacements ) => {
   const missed = replYaml.match( /OVH:<(.*)>/g )
 
   if ( missed ) {
-    console.log( '\n***ERROR: missing replacements in overwhelm.yaml for these tags:\n' )
+    log( '\n***ERROR: missing replacements in overwhelm.yaml for these tags:\n' )
     missed.forEach( ( miss ) => {
-      console.log( `     TAG: ${miss.match( /OVH:<(.*)>/ )[1]}` )
+      log( `     TAG: ${miss.match( /OVH:<(.*)>/ )[1]}` )
     } )
-    console.log( '' )
+    log( '' )
     process.exit( 1 )
   }
 
@@ -199,7 +195,7 @@ const buildYaml = ( yamls, replacements ) => {
 
   values.forEach( ( yaml ) => {
     if ( !fs.existsSync( yaml ) ) {
-      console.log( `\n***ERROR: yaml file missing => ${yaml}\n` )
+      log( `\n***ERROR: yaml file missing => ${yaml}\n` )
       process.exit( 1 )
     }
 
@@ -221,14 +217,11 @@ const values = buildYaml( yams, over )
 if ( args.key ) {
   const section = args.section || 'default'
   const keys = yaml[section]
-  if ( args.verbose ) {
-    console.log( 'replacements' )
-    console.log( keys )
-  }
+  debug( { keys }, '<==Replacements' )
   if ( args.export ) {
-    console.log( `export ${args.key}=${keys[args.key]}` )
+    log( `export ${args.key}=${keys[args.key]}` )
   } else {
-    console.log( keys[args.key] )
+    log( keys[args.key] )
   }
   if ( !args.continue ) process.exit()
 }
@@ -241,22 +234,19 @@ const opt = {
 
 const ahoy = glob.sync( '*/', opt )
 
-// .noahoy = skips generation of values.yaml but allows helmup to run
-// .nohelm = overwhelm and helmup will skip the deployment entirely
+// .nohelm = skips generation of values.yaml but allows helmup to run
 ahoy.forEach( ( dir ) => {
-  if ( fs.existsSync( `${dir}/.noahoy` ) ) {
-    !args.fetchctx && console.log( chalk.yellow( `***WARNING: .noahoy detected, not regenerating ${dir}values.yaml` ) )
-  } else if ( !fs.existsSync( `${dir}/.nohelm` ) ) {
-    args.verbose && console.log( `Generating values in ${dir}` ) // eslint-disable-line no-unused-expressions
+  if ( !fs.existsSync( `${dir}/.nohelm` ) ) {
+    // debug( { dir }, 'Generating values' ) // eslint-disable-line no-unused-expressions
 
     let valuesOut = [] // An accumulator for all of the output values.
 
     // Add support for localized values for things like keel (or whatever)
-    const valuesLocal = `${dir}values-local.yaml`
+    const valuesLocal = `${dir}/values-local.yaml`
 
     if ( fs.existsSync( valuesLocal ) ) {
-      args.verbose && console.log( `     Local values in ${valuesLocal}` ) // eslint-disable-line no-unused-expressions
-      valuesOut += '\n# AHOY => values-local.yaml\n\n'
+      debug( { valuesLocal }, 'rendering values.yaml' )
+      valuesOut += '\n# OVERWHELM => values-local.yaml\n\n'
       valuesOut += fs.readFileSync( valuesLocal, { encoding : 'utf-8' } )
     }
     valuesOut += `\n${values}`
@@ -283,7 +273,7 @@ ahoy.forEach( ( dir ) => {
     const valuesOverride = `${dir}/values-override.yaml`
 
     if ( fs.existsSync( valuesOverride ) ) {
-      console.log( `===>Applying Overrides=> ${valuesOverride}` )
+      log( `===>Applying Overrides=> ${valuesOverride}` )
       const overrides = YAML.load( fs.readFileSync( valuesOverride, { encofind : 'utf-8' } ) )
       const allValues = YAML.load( valuesOut )
       valuesOut = YAML.dump( deepmerge( allValues, overrides ), { quotingType : '"' } )
@@ -313,7 +303,8 @@ const projectId = over.PROJECT_ID
 const zone = over.GCE_REGION + ( over.GCE_ZONE ? `-${over.GCE_ZONE}` : '' )
 
 if ( !clusterName ) {
-  console.log( chalk.yellow(
+  /* eslint-disable quotes */
+  log( chalk.yellow(
     `
 
     The overwhelm.yaml file in this branch is missing a CLUSTER_NAME setting,
@@ -322,24 +313,25 @@ if ( !clusterName ) {
     adding this line following PROJECT_NAME in the default section of
     overwhelm.yaml:
 
-      ${chalk.bold.green( 'CLUSTER_NAME   : \'${PROJECT_NAME}\'' )}
+      ${chalk.bold.green( `CLUSTER_NAME   : '\${PROJECT_NAME}'` )}
 
     However, if the CLUSTER_NAME is not the same as the PROJECT_NAME then go
     ahead and add the CLUSTER_NAME setting to the ${chalk.bold.green( projectId )} section of the
     overwhelm.yaml file.
 
     ` ) )
+  /* eslint-enable quotes */
   process.exit( 1 )
 }
 
-let kubeout = '_no_current_context_'
+// the kubectl call will throw if there isn't an existing context
+let currentCtx = '_no_current_context_'
 try {
-  kubeout = exec( 'kubectl config current-context', { stdio : [ undefined ] } )
+  currentCtx = await shellCmd( 'kubectl config current-context' )
 } catch ( err ) {
-  // err && console.log( `Caught Error =>[${err}]` )
+  // err && log( `Caught Error =>[${err}]` )
 }
 
-const currentCtx = kubeout.toString().trim()
 const desiredCtx = `gke_${projectId}_${zone}_${clusterName}`
 const cmd = `gcloud container clusters get-credentials ${clusterName} --zone ${zone} --project ${projectId}`
 
@@ -353,7 +345,7 @@ if ( args.matchk8s ) {
 if ( args.fetchctx ) {
   if ( !matchedCtx ) {
     try {
-      exec( `${cmd}`, { stdio : [ undefined ] } )
+      await shellCmd( cmd )
     } catch ( err ) {
       process.exit( 61 ) // ECONNREFUSED
     }
@@ -362,13 +354,13 @@ if ( args.fetchctx ) {
 }
 
 if ( matchedCtx ) {
-  console.log( `
+  log( `
 
   Current context matches => ${chalk.bold.green( currentCtx )}
 
   ` )
 } else {
-  console.log( `
+  log( `
   You are about to change your Kubernetes context to:
       CLUSTER_NAME => ${chalk.bold.green( clusterName )}
       PROJECT_ID   => ${chalk.bold.green( projectId )}
@@ -380,18 +372,18 @@ const operation = matchedCtx ? 'refetch the credentials' : 'switch your context'
 const ans = ask.question( `Enter "${chalk.bold.green( 'yes' )}" if you wish to ${operation} => ` )
 if ( ans !== 'yes' ) {
   if ( matchedCtx ) {
-    console.log( chalk.bold.yellow( '\n\nSkipping a refetch of your Kubernetes context and credentials\n\n' ) )
+    log( chalk.bold.yellow( '\n\nSkipping a refetch of your Kubernetes context and credentials\n\n' ) )
     process.exit( 0 )
   }
 
-  console.log( chalk.bold.red( '\n\nYour Kubernetes context and credentials do NOT match - this could be bad!\n\n' ) )
+  log( chalk.bold.red( '\n\nYour Kubernetes context and credentials do NOT match - this could be bad!\n\n' ) )
   process.exit( 1 )
 }
 
-console.log( `\n\nExecuting: ${chalk.bold.yellow( cmd )}\n\n` )
-exec( `${cmd}`, ( error, out ) => {
-  if ( error ) {
-    console.log( `\n\n***ERROR ${error}\n\n` )
-    process.exit( 1 )
-  }
-} )
+log( `\n\nExecuting: ${chalk.bold.yellow( cmd )}\n\n` )
+try {
+  await shellCmd( cmd )
+} catch ( error ) {
+  log( { error, cmd }, '***ERROR overwhelm' )
+  process.exit( 1 )
+}

package/src/push-my-chart.mjs

@@ -1,17 +1,18 @@
 #!/usr/bin/env node
 
-import execa from 'execa'
 import { lt as semverLt } from 'semver'
 
+import { log, shellCmd } from './Utils.mjs'
+
 const pushItGood = async () => {
-  const version = await execa( 'helm', [ 'version', '--short' ] )
-  const pusher = ( semverLt( version.stdout, '3.7.0' ) ? 'push' : 'cm-push' )
+  const version = await shellCmd( 'helm version --short' )
+  const pusher = ( semverLt( version, '3.7.0' ) ? 'push' : 'cm-push' )
 
   try {
-    const museum = await execa( 'helm', [ pusher, 'helm', 'leverege' ] )
-    console.log( museum.stdout ) // eslint-disable-line no-console
+    const museum = await shellCmd( `helm ${pusher} helm leverege` )
+    log( museum )
   } catch ( ex ) {
-    console.log( ex.stderr ) // eslint-disable-line no-console
+    log( ex.stderr )
     process.exit( 1 )
   }
 }