cloud-mu 3.4.0 → 3.5.0

data/bin/mu-gcp-setup

@@ -44,6 +44,7 @@ Usage:
   opt :logs, "Ensure the presence of an Cloud Storage bucket prefixed with 'Mu_Logs' for use with CloudTrails, syslog, etc.", :require => false, :default => false, :type => :boolean
 #  opt :dns, "Ensure the presence of a private DNS Zone called for internal amongst Mu resources.", :require => false, :default => false, :type => :boolean
   opt :uploadlogs, "Push today's log files to the Cloud Storage bucket created by the -l option.", :require => false, :default => false, :type => :boolean
+  opt :optdisk, "Create a block volume for /opt and slide our installation onto it", :require => false, :default => false, :type => :boolean
 end
 
 if MU::Cloud::Google.hosted? and !$MU_CFG['google']
@@ -135,6 +136,38 @@ if $opts[:sg]
 
 end
 
+if $opts[:optdisk] and !File.open("/etc/mtab").read.match(/ \/opt[\s\/]/)
+  myname = MU::Cloud::Google.getGoogleMetaData("instance/name")
+  wd = Dir.getwd
+  Dir.chdir("/")
+  if File.exists?("/opt/opscode/bin/chef-server-ctl")
+    system("/opt/opscode/bin/chef-server-ctl stop")
+  end
+  if !File.exists?("/sbin/mkfs.xfs")
+    system("/usr/bin/yum -y install xfsprogs")
+  end
+  MU::Master.disk(myname+"-mu-opt", "/opt_tmp", 30)
+  uuid = MU::Master.diskUUID(myname+"-mu-opt")
+  if !uuid or uuid.empty?
+    MU.log "Failed to retrieve UUID of block device #{myname}-mu-opt", MU::ERR, details: MU::Cloud::AWS.realDevicePath(myname+"-mu-opt")
+    exit 1
+  end
+  MU.log "Moving contents of /opt to /opt_tmp", MU::NOTICE
+  system("/bin/mv /opt/* /opt_tmp/")
+  exit 1 if $?.exitstatus != 0
+  MU.log "Remounting /opt_tmp /opt", MU::NOTICE
+  system("/bin/umount /opt_tmp")
+  exit 1 if $?.exitstatus != 0
+  system("echo '#{uuid} /opt xfs defaults 0 0' >> /etc/fstab")
+  system("/bin/mount -a")
+  exit 1 if $?.exitstatus != 0
+  if File.exists?("/opt/opscode/bin/chef-server-ctl")
+    system("/opt/opscode/bin/chef-server-ctl start")
+  end
+  Dir.chdir(wd)
+end
+
+
 $bucketname = MU::Cloud::Google.adminBucketName
 
 if $opts[:logs]
@@ -204,8 +237,10 @@ if $opts[:logs]
         raise MuError, e.inspect
       end
     end
-# XXX stop doing this per-bucket, chowderhead
-    MU::Master.disk("/dev/xvdl", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
+
+    myname = MU::Cloud::Google.getGoogleMetaData("instance/name")
+    MU::Master.disk("/dev/"+myname+"-mu-logs", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
+
   }
 
 end

data/bin/mu-node-manage

@@ -188,6 +188,9 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false,
       nodeclasses.each_pair { |nodeclass, servers|
         servers.each_pair { |mu_name, server|
           next if nodes.size > 0 and !nodes.include?(mu_name)
+          server.myFirewallRules.each { |fw|
+            fw.groom
+          }
           count = count + 1
           child = Process.fork {
             begin

data/bin/mu-refresh-ssl

@@ -0,0 +1,67 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ARGV.size > 0
+  puts "#{$0}: Freshen the Mu Master's API and configuration documentation"
+  exit 1
+end
+
+require 'rubygems'
+require 'bundler/setup'
+require 'erb'
+require 'tempfile'
+require 'fileutils'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+# XXX need special handling if Mu_CA.pem is expired
+
+ca = %w{Mu_CA}
+services = %w{rsyslog mommacat ldap consul vault}
+chef_server = %w{Mu_CA mommacat}
+
+certfiles = Dir.glob(MU.dataDir+"/ssl/*.pem")
+certfiles.concat(Dir.glob(MU.dataDir+"/ssl/*.crt"))
+
+now= Time.now
+
+need_chef_ssl_run = false
+need_chef_reconfigure = false
+
+certfiles.each { |filename|
+  shortname = filename.gsub(/.*?\/|\.(pem|crt)$/, '')
+  cert = OpenSSL::X509::Certificate.new(File.read(filename))
+  if cert.not_after < now
+    need_chef_ssl_run = true if services.include?(shortname)
+    need_chef_reconfigure = true if chef_server.include?(shortname)
+    newpath = filename+"."+now.strftime("%Y%m%d%H").to_s
+    MU.log filename+" is expired, archiving to #{newpath} and regenerating", MU::WARN
+    File.rename(filename, newpath)
+  end
+}
+
+
+if need_chef_ssl_run
+# XXX need to temporarily disable ssl checks in knife.rb and client.rb for this, maybe?
+  if !system("/opt/chef/bin/chef-client -o 'recipe[mu-master::ssl-certs]'")
+    MU.log "Got bad exit code trying to run recipe[mu-master::ssl-certs]'", MU::ERR
+    exit 1
+  end
+end
+
+if need_chef_reconfigure
+  system("CHEF_LICENSE=\"accept\" /opt/opscode/bin/chef-server-ctl reconfigure")
+  system("CHEF_LICENSE=\"accept\" /opt/opscode/bin/chef-server-ctl restart")
+end

data/bin/mu-run-tests

@@ -37,13 +37,23 @@ Usage:
   opt :max_retries, "Number of times to retry failed tests in --dryrun mode.", :require => false, :default => 2, :type => :integer
   opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false
   opt :verbose, "Show more information while running", :require => false, :default => false
+  opt :clouds, "Select a subset of support cloud providers on which to test", :required => false, :type => :strings, :default => MU::Cloud.availableClouds.reject { |c| c == "CloudFormation" }
 end
 
 only = ARGV
 
 files = Dir.glob("*.yaml", base: dir)
 files.concat(Dir.glob("*.yml", base: dir))
-baseclouds = MU::Cloud.availableClouds.reject { |c| c == "CloudFormation" }
+valid_clouds = MU::Cloud.availableClouds.reject { |c| c == "CloudFormation" }
+baseclouds = []
+$opts[:clouds].each { |cloud|
+  if !valid_clouds.include?(cloud)
+    MU.log "'#{cloud}' isn't one of my available cloud providers", MU::ERR, details: valid_clouds
+  else
+    baseclouds << cloud
+  end
+}
+
 
 commands = {}
 failures = []
@@ -96,7 +106,7 @@ files.each { |f|
         conf_engine = MU::Config.new(f, cloud: cloud)
       rescue StandardError => e
         MU.log e.message+" parsing "+f+" with cloud "+cloud, MU::WARN, details: e.backtrace
-        failures << f+" ("+commands[cmd]["cloud"]+")"
+        failures << f+" ["+commands[cmd]["cloud"]+"] - "+e.class.name+"\n\t"+e.message.gsub(/\n/, "\t\n")
         next 
       end
       parsed = MU::Config.stripConfig(conf_engine.config)
@@ -175,7 +185,7 @@ commands.keys.each { |cmd|
       results[cmd_thr]["failed"] << "main"
     end
 
-    if $opts[:full] and results[cmd_thr]["output"].match(/deploy - Deployment id: .*? \((.*?)\)/)
+    if $opts[:full] and results[cmd_thr]["output"].to_s.match(/deploy - Deployment id: .*? \((.*?)\)/)
       deploy_id = Regexp.last_match[1]
       adoptdir = Dir.mktmpdir(commands[cmd_thr]["file"].gsub(/[^a-z0-9]|yaml$/i, ""))
       if commands[cmd_thr]["types"] and commands[cmd_thr]["types"].size > 0
@@ -227,6 +237,6 @@ results.keys.sort { |a, b|
 }
 
 if failures.size > 0
-  puts "\n#{failures.size.to_s.bold} failure#{failures.size == 1 ? "" : "s"} in "+failures.uniq.map { |f| f.light_red }.join(", ")
+  puts "\n#{failures.size.to_s.bold} failure#{failures.size == 1 ? "" : "s"} in:\n"+failures.uniq.map { |f| f.light_red }.join("\n")
   exit 1
 end

data/bin/mu-self-update

@@ -156,6 +156,15 @@ fi
 git config branch.${branch}.remote origin
 git config branch.${branch}.merge refs/heads/$branch
 
+CHEF_CLIENT_VERSION=`grep '^CHEF_CLIENT_VERSION=' /opt/mu/lib/install/installer |cut -d\" -f2`
+
+# Make sure any new bootstrappy stuff has been applied
+chef-apply /opt/mu/lib/cookbooks/mu-master/recipes/init.rb
+
+# Make sure bundler will use a recent git binary
+if [ -d /usr/local/git-current ];then
+  export PATH="/usr/local/git-current/bin:${PATH}"
+fi
 
 if [ "`diff -r $MU_LIBDIR/cookbooks $MU_DATADIR/tmp/cookbook_changes.$$`" != "" ];then
 	rebuild_chef_artifacts=1
@@ -177,20 +186,31 @@ set -e
 echo "${GREEN}Cleaning gems in ${BOLD}/usr/local/ruby-current${NORM}${GREEN}${NORM}"
 cd $MU_LIBDIR/modules
 /usr/local/ruby-current/bin/bundle update
+set +e
 /usr/local/ruby-current/bin/bundle clean --force
+set -e
 cd
 
-chef_major="`/opt/chef/bin/chef-apply --version | awk '{print $2}' | cut -d\. -f1`"
-if [ "$chef_major" == "12" ];then
-  DIST_VERSION=`rpm -qa \*-release\* | grep -Ei "redhat|^centos" | cut -d"-" -f3`
-  # IS_AMAZON=0
-  if [ "$DIST_VERSION" == "" ];then # funny package name in Amazon Linux
-    DIST_VERSION=6
-  # IS_AMAZON=1
-  elif [ "$DIST_VERSION" == "server" ];then # funny package name in RHEL6
-    DIST_VERSION="6"
+DIST_VERSION=`rpm -qa \*-release\* | grep -Ei "redhat|^centos" | cut -d"-" -f3`
+# IS_AMAZON=0
+if [ "$DIST_VERSION" == "" ];then # funny package name in Amazon Linux
+  DIST_VERSION=6
+# IS_AMAZON=1
+elif [ "$DIST_VERSION" == "server" ];then # funny package name in RHEL6
+  DIST_VERSION="6"
+else
+  DIST_VERSION="7"
+fi
+
+grep ^chef_license /etc/chef/client.rb || echo "chef_license 'accept'" >> /etc/chef/client.rb
+
+if ! rpm -q chef;then
+  yes | rpm -ivh https://packages.chef.io/files/stable/chef/${CHEF_CLIENT_VERSION}/el/${DIST_VERSION}/chef-${CHEF_CLIENT_VERSION}-1.el${DIST_VERSION}.x86_64.rpm
+else
+  cur_chef_ver="`rpm -q chef | cut -d\- -f2`"
+  if [ "${cur_chef_ver}" != "${CHEF_CLIENT_VERSION}" ];then
+    yes | rpm -Uvh https://packages.chef.io/files/stable/chef/${CHEF_CLIENT_VERSION}/el/${DIST_VERSION}/chef-${CHEF_CLIENT_VERSION}-1.el${DIST_VERSION}.x86_64.rpm
   fi
-  rpm -Uvh https://packages.chef.io/files/stable/chef/14.13.11/el/${DIST_VERSION}/chef-14.13.11-1.el${DIST_VERSION}.x86_64.rpm
 fi
 
 /opt/chef/bin/chef-apply $MU_LIBDIR/cookbooks/mu-master/recipes/init.rb

data/bin/mu-upload-chef-artifacts

@@ -30,6 +30,10 @@ if [ -z $MU_CHEF_CACHE ];then
 fi
 manifest="$MU_CHEF_CACHE/mu_manifest"
 berksdir="$HOMEDIR/.berkshelf"
+knife_cfg=""
+if [ -f "$HOMEDIR/.chef/knife.rb" ];then
+  knife_cfg="-c $HOMEDIR/.chef/knife.rb"
+fi
 
 rm -rf "${berksdir}/" # Just... don't trust it to check cache correctly
 
@@ -190,7 +194,7 @@ add_berkshelf_cookbooks()
     cd $repodir
     set +e
     for name in $berkshelf_cookbooks;do
-      $knife cookbook delete $name --yes -a
+      $knife cookbook $knife_cfg delete $name --yes -a
     done
     set -e
   fi
@@ -361,24 +365,24 @@ fi
 if [ "$nopurge" == "" -a "$all" == "1" ];then
   if [ "$use_on_disk" == "1" -o "$all" == "1" ];then
     if [ "$cookbooks_only" == "1" ];then
-      $knife cookbook bulk delete --purge '.+' --yes 2>/dev/null
+      $knife cookbook $knife_cfg bulk delete --purge '.+' --yes 2>/dev/null
       /bin/rm -rf $MU_CHEF_CACHE/cookbooks $MU_CHEF_CACHE/site_cookbooks
     elif [ "$bags_only" == "1" ];then
       # Nowadays we have data bags that are persistent and node-related. Leave
       # them be, and only delete our automatic ones.
       for bag in nagios_users nagios_servers demo;do
-        $knife data bag delete $bag --yes 2>/dev/null
+        $knife data bag $knife_cfg delete $bag --yes 2>/dev/null
       done
     else
-      $knife cookbook bulk delete --purge '.+' --yes 2>/dev/null
+      $knife cookbook $knife_cfg bulk delete --purge '.+' --yes 2>/dev/null
       /bin/rm -rf $MU_CHEF_CACHE/cookbooks $MU_CHEF_CACHE/site_cookbooks
       /bin/rm -rf $MU_CHEF_CACHE/roles $MU_CHEF_CACHE/environments $MU_CHEF_CACHE/data_bags
-      $knife role bulk delete '.*' --yes 2>/dev/null
-      for env in `$knife environment list | grep -v '_default$'`;do
-        $knife environment delete $env --yes 2>/dev/null
+      $knife role $knife_cfg bulk delete '.*' --yes 2>/dev/null
+      for env in `$knife environment $knife_cfg list | grep -v '_default$'`;do
+        $knife environment $knife_cfg delete $env --yes 2>/dev/null
       done
       for bag in nagios_users nagios_servers demo;do
-        $knife data bag delete $bag --yes 2>/dev/null
+        $knife data bag $knife_cfg delete $bag --yes 2>/dev/null
       done
     fi
   fi
@@ -411,16 +415,16 @@ for repo in $REPOS;do
       set +e
       if [ "$type" == "cookbooks" -o "$type" == "site_cookbooks" ];then
         /bin/rm -rf $MU_CHEF_CACHE/$type/$name
-        $knife cookbook delete $name --yes --all
+        $knife cookbook $knife_cfg delete $name --yes --all
       elif [ "$type" == "roles" -a "$cookbooks_only" == "" ];then
         /bin/rm -rf $MU_CHEF_CACHE/$type/$name.json
-        $knife role delete $name --yes
+        $knife role $knife_cfg delete $name --yes
       elif [ "$type" == "environments" -a "$cookbooks_only" == "" ];then
         /bin/rm -rf $MU_CHEF_CACHE/$type/$name.json
-        $knife environment delete $name --yes
+        $knife environment $knife_cfg delete $name --yes
       elif [ "$type" == "data_bags" -a "$cookbooks_only" == "" ];then
         /bin/rm -rf $MU_CHEF_CACHE/$type/$name
-        $knife data bag delete $name --yes
+        $knife data bag $knife_cfg delete $name --yes
       fi
       set -e
     done
@@ -621,9 +625,9 @@ if [ -d "$MU_DATADIR/users" -a "$USER" == "root" ];then
     if [ -f "$bagdir/$admin.json" ];then
       if [ "$bagcreated" == "0" ];then
         bagcreated=1
-        $knife data bag create nagios_users
+        $knife data bag $knife_cfg create nagios_users
       fi
-      $knife data bag from file nagios_users $bagdir/$admin.json
+      $knife data bag $knife_cfg from file nagios_users $bagdir/$admin.json
     fi
     if [ ! -f "$bagdir/$admin.json" ];then
       id="`echo $admin | sed -e 's/@/_/'`"
@@ -657,21 +661,21 @@ if [ "$all" != "1" ];then
         if [ "$match" == "" -o "$match" == "$itemname" ];then
           if [ "$bagcreated" == "0" ];then
             bagcreated=1
-            $knife data bag create $bag
+            $knife data bag $knife_cfg create $bag
           fi
-          $knife data bag from file $bag $file
+          $knife data bag $knife_cfg from file $bag $file
         fi
       done
     done
   else
     for role in $upload_roles;do
       if [ "$match" == "" -o "$match" == "$role" ];then
-        $knife role from file $MU_CHEF_CACHE/roles/$role.json
+        $knife role $knife_cfg from file $MU_CHEF_CACHE/roles/$role.json
       fi
     done
     for env in $upload_environments;do
       if [ "$match" == "" -o "$match" == "$env" ];then
-        $knife environment from file $MU_CHEF_CACHE/environments/$env.json
+        $knife environment $knife_cfg from file $MU_CHEF_CACHE/environments/$env.json
       fi
     done
     set +e
@@ -682,9 +686,9 @@ if [ "$all" != "1" ];then
         if [ "$match" == "" -o "$match" == "$itemname" ];then
           if [ "$bagcreated" == "0" ];then
             bagcreated=1
-            $knife data bag create $bag
+            $knife data bag $knife_cfg create $bag
           fi
-          $knife data bag from file $bag $file
+          $knife data bag $knife_cfg from file $bag $file
         fi
       done
     done
@@ -713,9 +717,9 @@ else
         if [ "$match" == "" -o "$match" == "$itemname" ];then
           if [ "$bagcreated" == "0" ];then
             bagcreated=1
-            $knife data bag create $folder
+            $knife data bag $knife_cfg create $folder
           fi
-          $knife data bag from file $folder $file
+          $knife data bag $knife_cfg from file $folder $file
         fi
       done
     done
@@ -729,7 +733,7 @@ else
     for file in $_files;do
       role="`echo $file | sed -r 's/.*\/([^\\]+).json$/\1/'`"
       if [ "$match" == "" -o "$match" == "$role" ];then
-        $knife role from file $file
+        $knife role $knife_cfg from file $file
       fi
     done
     
@@ -737,7 +741,7 @@ else
     for file in $_files;do
       env="`echo $file | sed -r 's/.*\/([^\\]+).json$/\1/'`"
       if [ "$match" == "" -o "$match" == "$env" ];then
-        $knife environment from file $file
+        $knife environment $knife_cfg from file $file
       fi
     done
     
@@ -751,9 +755,9 @@ else
         if [ "$match" == "" -o "$match" == "$itemname" ];then
           if [ "$bagcreated" == "0" ];then
             bagcreated=1
-            $knife data bag create $folder
+            $knife data bag $knife_cfg create $folder
           fi
-          $knife data bag from file $folder $file
+          $knife data bag $knife_cfg from file $folder $file
         fi
       done
       set -e

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.4.0'
-  s.date        = '2020-10-22'
+  s.version     = '3.5.0'
+  s.date        = '2021-01-18'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -31,19 +31,21 @@ EOF
   s.authors     = ["John Stange", "Robert Patt-Corner", "Ryan Bolyard", "Zach Rowe"]
   s.email       = 'eGTLabs@eglobaltech.com'
   s.files       = build_file_list(whereami)
-  s.executables = Dir.entries(whereami+"/bin").reject { |f| File.directory?(f) }
+  if Dir.exists?(whereami+"/bin")
+    s.executables = Dir.entries(whereami+"/bin").reject { |f| File.directory?(f) }
+  end
   s.homepage    =
     'https://github.com/cloudamatic/mu'
   s.license       = 'BSD-3-Clause-Attribution'
   s.add_runtime_dependency 'addressable', '~> 2.5'
   s.add_runtime_dependency "aws-sdk", "~> 3.0"
-  s.add_runtime_dependency 'azure_sdk', "~> 0.52"
-  s.add_runtime_dependency 'bundler', "~> 1.17"
+  s.add_runtime_dependency 'azure_sdk', '~> 0.65'
+  s.add_runtime_dependency 'bundler', "~> 2.1.4"
   s.add_runtime_dependency 'chronic_duration', "~> 0.10"
   s.add_runtime_dependency 'color', "~> 1.8"
   s.add_runtime_dependency 'colorize', "~> 0.8"
   s.add_runtime_dependency 'erubis', "~> 2.7"
-  s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
+  s.add_runtime_dependency 'google-api-client', "~> 0.50.0"
   s.add_runtime_dependency 'googleauth', "~> 0.6"
   s.add_runtime_dependency 'inifile', "~> 3.0"
   s.add_runtime_dependency 'json-schema', "~> 2.8"

data/cookbooks/mu-master/attributes/default.rb

@@ -22,6 +22,9 @@ default['apache']['contact'] = $MU_CFG['mu_admin_email']
 default['apache']['traceenable'] = 'Off'
 
 default["apache"]["listen"] = ["*:80", "*:443", "*:8443"]
+default['apache']['user'] = "apache"
+default['apache']['group'] = "apache"
+
 
 override["nagios"]["http_port"] = 8443
 default['nagios']['enable_ssl'] = true
@@ -47,6 +50,7 @@ default["nagios"]["log_dir"] = "/var/log/httpd"
 default['nagios']['cgi-bin'] = "/usr/lib/cgi-bin/"
 default['nagios']['cgi-path'] = "/nagios/cgi-bin/"
 default['nagios']['server_role'] = "mu-master"
+default['nagios']['group'] = "nagios"
 default['nagios']['server']['install_method'] = 'source'
 default['nagios']['multi_environment_monitoring'] = true
 default['nagios']['users_databag'] = "nagios_users"
@@ -94,7 +98,7 @@ case node['platform']
     ssh_user = "ec2-user"
 end
 
-default['application_attributes']['sshd_allow_groups'] = "#{ssh_user} mu-users"
+default['application_attributes']['sshd_allow_groups'] = "#{ssh_user} mu-users adm google-sudoers"
 default['application_attributes']['sshd_allow_password_auth'] = true
 default['update_nagios_only'] = false
 default['apache']['listen'] = [80, 443, 8443]