cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/groomer.rb

@@ -18,7 +18,7 @@ module MU
   class Groomer
 
     # An exception denoting a Groomer run that has failed
-    class RunError < MuError
+    class RunError < StandardError
     end
 
     # An exception denoting nonexistent secret
@@ -49,6 +49,7 @@ module MU
     # @param groomer [String]: The grooming agent to load.
     # @return [Class]: The class object implementing this groomer agent
     def self.loadGroomer(groomer)
+      return nil if !groomer
       if !File.size?(MU.myRoot+"/modules/mu/groomers/#{groomer.downcase}.rb")
         raise MuError, "Requested to use unsupported grooming agent #{groomer}"
       end

data/modules/mu/groomers/ansible.rb

@@ -20,11 +20,17 @@ module MU
     # Support for Ansible as a host configuration management layer.
     class Ansible
 
+      # Failure to load or create a deploy
+      class NoAnsibleExecError < MuError;
+      end
 
-      # Location in which we'll find our Ansible executables
+      # Location in which we'll find our Ansible executables. This only applies
+      # to full-grown Mu masters; minimalist gem installs will have to make do
+      # with whatever Ansible executables they can find in $PATH.
       BINDIR = "/usr/local/python-current/bin"
       @@pwfile_semaphore = Mutex.new
 
+
       # @param node [MU::Cloud::Server]: The server object on which we'll be operating
       def initialize(node)
         @config = node.config
@@ -32,9 +38,14 @@ module MU
         @inventory = Inventory.new(node.deploy)
         @mu_user = node.deploy.mu_user
         @ansible_path = node.deploy.deploy_dir+"/ansible"
+        @ansible_execs = MU::Groomer::Ansible.ansibleExecDir
+
+        if !@ansible_execs or @ansible_execs.empty?
+          raise NoAnsibleExecError, "No Ansible executables found in visible paths"
+        end
 
         [@ansible_path, @ansible_path+"/roles", @ansible_path+"/vars", @ansible_path+"/group_vars", @ansible_path+"/vaults"].each { |dir|
-          if !Dir.exists?(dir)
+          if !Dir.exist?(dir)
             MU.log "Creating #{dir}", MU::DEBUG
             Dir.mkdir(dir, 0755)
           end
@@ -77,19 +88,20 @@ module MU
         end
         path = dir+"/"+item
 
-        if !Dir.exists?(dir)
+        if !Dir.exist?(dir)
           FileUtils.mkdir_p(dir, mode: 0700)
         end
 
-        if File.exists?(path)
+        if File.exist?(path)
           MU.log "Overwriting existing vault #{vault} item #{item}"
         end
         File.open(path, File::CREAT|File::RDWR|File::TRUNC, 0600) { |f|
           f.write data
         }
-        cmd = %Q{#{BINDIR}/ansible-vault encrypt #{path} --vault-id #{pwfile}}
+
+        cmd = %Q{#{ansibleExecDir}/ansible-vault encrypt #{path} --vault-password-file #{pwfile}}
         MU.log cmd
-        system(cmd)
+        raise MuError, "Failed Ansible command: #{cmd}" if !system(cmd)
       end
 
       # see {MU::Groomer::Ansible.saveSecret}
@@ -110,17 +122,17 @@ module MU
 
         pwfile = vaultPasswordFile
         dir = secret_dir+"/"+vault
-        if !Dir.exists?(dir)
+        if !Dir.exist?(dir)
           raise MuNoSuchSecret, "No such vault #{vault}"
         end
 
         data = nil
         if item
           itempath = dir+"/"+item
-          if !File.exists?(itempath)
+          if !File.exist?(itempath)
             raise MuNoSuchSecret, "No such item #{item} in vault #{vault}"
           end
-          cmd = %Q{#{BINDIR}/ansible-vault view #{itempath} --vault-id #{pwfile}}
+          cmd = %Q{#{ansibleExecDir}/ansible-vault view #{itempath} --vault-password-file #{pwfile}}
           MU.log cmd
           a = `#{cmd}`
           # If we happen to have stored recognizeable JSON, return it as parsed,
@@ -158,14 +170,14 @@ module MU
           raise MuError, "Must call deleteSecret with at least a vault name"
         end
         dir = secret_dir+"/"+vault
-        if !Dir.exists?(dir)
+        if !Dir.exist?(dir)
           raise MuNoSuchSecret, "No such vault #{vault}"
         end
 
         data = nil
         if item
           itempath = dir+"/"+item
-          if !File.exists?(itempath)
+          if !File.exist?(itempath)
             raise MuNoSuchSecret, "No such item #{item} in vault #{vault}"
           end
           MU.log "Deleting Ansible vault #{vault} item #{item}", MU::NOTICE
@@ -189,13 +201,28 @@ module MU
       # @param output [Boolean]: Display Ansible's regular (non-error) output to the console
       # @param override_runlist [String]: Use the specified run list instead of the node's configured list
       def run(purpose: "Ansible run", update_runlist: true, max_retries: 5, output: true, override_runlist: nil, reboot_first_fail: false, timeout: 1800)
+        bootstrap
         pwfile = MU::Groomer::Ansible.vaultPasswordFile
         stashHostSSLCertSecret
 
-        cmd = %Q{cd #{@ansible_path} && #{BINDIR}/ansible-playbook -i hosts #{@server.config['name']}.yml --limit=#{@server.mu_name} --vault-id #{pwfile} --vault-id #{@ansible_path}/.vault_pw}
+        ssh_user = @server.config['ssh_user'] || "root"
 
-        MU.log cmd
-        system(cmd)
+        cmd = %Q{cd #{@ansible_path} && #{@ansible_execs}/ansible-playbook -i hosts #{@server.config['name']}.yml --limit=#{@server.mu_name} --vault-password-file #{pwfile} --vault-password-file #{@ansible_path}/.vault_pw -u #{ssh_user}}
+
+        retries = 0
+        begin
+          MU.log cmd
+          raise MU::Groomer::RunError, "Failed Ansible command: #{cmd}" if !system(cmd)
+        rescue MU::Groomer::RunError => e
+          if retries < max_retries
+            sleep 30
+            retries += 1
+            MU.log "Failed Ansible run, will retry (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE, details: cmd
+            retry
+          else
+            raise MuError, "Failed Ansible command: #{cmd}"
+          end
+        end
       end
 
       # This is a stub; since Ansible is effectively agentless, this operation
@@ -224,6 +251,10 @@ module MU
           play["roles"] = @server.config['run_list']
         end
 
+        if @server.config['ansible_vars']
+          play["vars"] = @server.config['ansible_vars']
+        end
+
         File.open(@ansible_path+"/"+@server.config['name']+".yml", File::CREAT|File::RDWR|File::TRUNC, 0600) { |f|
           f.flock(File::LOCK_EX)
           f.puts [play].to_yaml
@@ -237,12 +268,13 @@ module MU
         @server.describe(update_cache: true) # Make sure we're fresh
 
         allvars = {
-          "deployment" => @server.deploy.deployment,
-          "service_name" => @config["name"],
-          "windows_admin_username" => @config['windows_admin_username'],
-          "mu_environment" => MU.environment.downcase,
+          "mu_deployment" => MU.structToHash(@server.deploy.deployment),
+          "mu_service_name" => @config["name"],
+          "mu_canonical_ip" => @server.canonicalIP,
+          "mu_admin_email" => $MU_CFG['mu_admin_email'],
+          "mu_environment" => MU.environment.downcase
         }
-        allvars['deployment']['ssh_public_key'] = @server.deploy.ssh_public_key
+        allvars['mu_deployment']['ssh_public_key'] = @server.deploy.ssh_public_key
 
         if @server.config['cloud'] == "AWS"
           allvars["ec2"] = MU.structToHash(@server.cloud_desc, stringify_keys: true)
@@ -262,7 +294,7 @@ module MU
           f.flock(File::LOCK_UN)
         }
 
-        groupvars = {}
+        groupvars = allvars.dup
         if @server.deploy.original_config.has_key?('parameters')
           groupvars["mu_parameters"] = @server.deploy.original_config['parameters']
         end
@@ -312,18 +344,41 @@ module MU
       # @param for_user [String]: Encrypt using the Vault password of the specified Mu user
       def self.encryptString(name, string, for_user = nil)
         pwfile = vaultPasswordFile
-        cmd = %Q{#{BINDIR}/ansible-vault}
-        system(cmd, "encrypt_string", string, "--name", name, "--vault-id", pwfile)
+        cmd = %Q{#{ansibleExecDir}/ansible-vault}
+        if !system(cmd, "encrypt_string", string, "--name", name, "--vault-password-file", pwfile)
+          raise MuError, "Failed Ansible command: #{cmd} encrypt_string <redacted> --name #{name} --vault-password-file"
+        end
       end
 
       private
 
+      def self.ansibleExecDir
+        path = nil
+        if File.exist?(BINDIR+"/ansible-playbook")
+          path = BINDIR
+        else
+          ENV['PATH'].split(/:/).each { |bindir|
+            if File.exist?(bindir+"/ansible-playbook")
+              path = bindir
+              if !File.exist?(bindir+"/ansible-vault")
+                MU.log "Found ansible-playbook executable in #{bindir}, but no ansible-vault. Vault functionality will not work!", MU::WARN
+              end
+              if !File.exist?(bindir+"/ansible-galaxy")
+                MU.log "Found ansible-playbook executable in #{bindir}, but no ansible-galaxy. Automatic community role fetch will not work!", MU::WARN
+              end
+              break
+            end
+          }
+        end
+        path
+      end
+
       # Get the +.vault_pw+ file for the appropriate user. If it doesn't exist,
       # generate one.
       def self.vaultPasswordFile(for_user = nil, pwfile: nil)
         pwfile ||= secret_dir(for_user)+"/.vault_pw"
         @@pwfile_semaphore.synchronize {
-          if !File.exists?(pwfile)
+          if !File.exist?(pwfile)
             MU.log "Generating Ansible vault password file at #{pwfile}", MU::DEBUG
             File.open(pwfile, File::CREAT|File::RDWR|File::TRUNC, 0400) { |f|
               f.write Password.random(12..14)
@@ -341,7 +396,7 @@ module MU
       # Figure out where our main stash of secrets is, and make sure it exists
       def self.secret_dir(user = MU.mu_user)
         path = MU.dataDir(user) + "/ansible-secrets"
-        Dir.mkdir(path, 0755) if !Dir.exists?(path)
+        Dir.mkdir(path, 0755) if !Dir.exist?(path)
 
         path
       end
@@ -350,6 +405,7 @@ module MU
       # artifacts, since 'roles' is an awfully generic name for a directory.
       # Short of a full, slow syntax check, this is the best we're liable to do.
       def isAnsibleRole?(path)
+        begin
         Dir.foreach(path) { |entry|
           if File.directory?(path+"/"+entry) and
              ["tasks", "vars"].include?(entry)
@@ -358,6 +414,8 @@ module MU
             return false
           end
         }
+        rescue Errno::ENOTDIR
+        end
         false
       end
 
@@ -368,20 +426,34 @@ module MU
 
         canon_links = {}
 
+        repodirs = []
+
+        # Make sure we search the global ansible_dir, if any is set
+        if $MU_CFG and $MU_CFG['ansible_dir'] and !$MU_CFG['ansible_dir'].empty?
+          if !Dir.exist?($MU_CFG['ansible_dir'])
+            MU.log "Config lists an Ansible directory at #{$MU_CFG['ansible_dir']}, but I see no such directory", MU::WARN
+          else
+            repodirs << $MU_CFG['ansible_dir']
+          end
+        end
+
         # Hook up any Ansible roles listed in our platform repos
         $MU_CFG['repos'].each { |repo|
           repo.match(/\/([^\/]+?)(\.git)?$/)
           shortname = Regexp.last_match(1)
-          repodir = MU.dataDir + "/" + shortname
+          repodirs << MU.dataDir + "/" + shortname
+        }
+
+        repodirs.each { |repodir|
           ["roles", "ansible/roles"].each { |subdir|
-            next if !Dir.exists?(repodir+"/"+subdir)
+            next if !Dir.exist?(repodir+"/"+subdir)
             Dir.foreach(repodir+"/"+subdir) { |role|
               next if [".", ".."].include?(role)
               realpath = repodir+"/"+subdir+"/"+role
               link = roledir+"/"+role
               
               if isAnsibleRole?(realpath)
-                if !File.exists?(link)
+                if !File.exist?(link)
                   File.symlink(realpath, link)
                   canon_links[role] = realpath
                 elsif File.symlink?(link)
@@ -402,16 +474,16 @@ module MU
         # Now layer on everything bundled in the main Mu repo
         Dir.foreach(MU.myRoot+"/ansible/roles") { |role|
           next if [".", ".."].include?(role)
-          next if File.exists?(roledir+"/"+role)
+          next if File.exist?(roledir+"/"+role)
           File.symlink(MU.myRoot+"/ansible/roles/"+role, roledir+"/"+role)
         }
 
         if @server.config['run_list']
           @server.config['run_list'].each { |role|
             found = false
-            if !File.exists?(roledir+"/"+role)
+            if !File.exist?(roledir+"/"+role)
               if role.match(/[^\.]\.[^\.]/) and @server.config['groomer_autofetch']
-                system(%Q{#{BINDIR}/ansible-galaxy}, "--roles-path", roledir, "install", role)
+                system(%Q{#{@ansible_execs}/ansible-galaxy}, "--roles-path", roledir, "install", role)
                 found = true
 # XXX check return value
               else
@@ -455,7 +527,7 @@ module MU
         def initialize(deploy)
           @deploy = deploy
           @ansible_path = @deploy.deploy_dir+"/ansible"
-          if !Dir.exists?(@ansible_path)
+          if !Dir.exist?(@ansible_path)
             Dir.mkdir(@ansible_path, 0755)
           end
 
@@ -528,7 +600,7 @@ module MU
 
         def read
           @inv = {}
-          if File.exists?(@ansible_path+"/hosts")
+          if File.exist?(@ansible_path+"/hosts")
             section = nil
             File.readlines(@ansible_path+"/hosts").each { |l|
               l.chomp!

data/modules/mu/groomers/chef.rb

@@ -71,7 +71,7 @@ module MU
             require 'chef/knife/bootstrap_windows_winrm'
             require 'chef/knife/bootstrap_windows_ssh'
             ::Chef::Config[:chef_server_url] = "https://#{MU.mu_public_addr}:7443/organizations/#{user}"
-            if File.exists?("#{Etc.getpwnam(mu_user).dir}/.chef/knife.rb")
+            if File.exist?("#{Etc.getpwnam(mu_user).dir}/.chef/knife.rb")
               MU.log "Loading Chef configuration from #{Etc.getpwnam(mu_user).dir}/.chef/knife.rb", MU::DEBUG
               ::Chef::Config.from_file("#{Etc.getpwnam(mu_user).dir}/.chef/knife.rb")
             end
@@ -218,7 +218,7 @@ module MU
         loadChefLib
         raise MuError, "No vault specified, nothing to delete" if vault.nil?
         MU.log "Deleting #{vault}:#{item} from vaults"
-        knife_db = nil
+
         knife_cmds = []
         if item.nil?
           knife_cmds << ::Chef::Knife::DataBagDelete.new(['data', 'bag', 'delete', vault])
@@ -270,7 +270,7 @@ module MU
 
         retries = 0
         try_upgrade = false
-        output = []
+        output_lines = []
         error_signal = "CHEF EXITED BADLY: "+(0...25).map { ('a'..'z').to_a[rand(26)] }.join
         runstart = nil
         cmd = nil
@@ -294,9 +294,26 @@ module MU
             Timeout::timeout(timeout) {
               retval = ssh.exec!(cmd) { |ch, stream, data|
                 puts data
-                output << data
-                raise MU::Cloud::BootstrapTempFail if data.match(/REBOOT_SCHEDULED| WARN: Reboot requested:/)
-                raise MU::Groomer::RunError, output.grep(/ ERROR: /).last if data.match(/#{error_signal}/)
+                output_lines << data
+                raise MU::Cloud::BootstrapTempFail if data.match(/REBOOT_SCHEDULED| WARN: Reboot requested:|Rebooting server at a recipe's request|Chef::Exceptions::Reboot/)
+                if data.match(/#{error_signal}/)
+                  error_msg = ""
+                  clip = false
+                  output_lines.each { |chunk|
+                    chunk.split(/\n/).each { |line|
+                      if !clip and line.match(/^========+/)
+                        clip = true
+                      elsif clip and line.match(/^Running handlers:/)
+                        break
+                      end
+
+                      if clip and line.match(/[a-z0-9]/)
+                        error_msg += line.gsub(/\e\[(\d+)m/, '')+"\n"
+                      end
+                    }
+                  }
+                  raise MU::Groomer::RunError, error_msg
+                end
               }
             }
           else
@@ -314,7 +331,7 @@ module MU
             if try_upgrade
               pp winrm.run("Invoke-WebRequest -useb https://omnitruck.chef.io/install.ps1 | Invoke-Expression; Install-Project -version:#{MU.chefVersion} -download_directory:$HOME")
             end
-            output = []
+            output_lines = []
             cmd = "c:/opscode/chef/bin/chef-client.bat --color"
             if override_runlist
               cmd = cmd + " -o '#{override_runlist}'"
@@ -324,20 +341,20 @@ module MU
               resp = winrm.run(cmd) do |stdout, stderr|
                 if stdout
                   print stdout if output
-                  output << stdout
+                  output_lines << stdout
                 end
                 if stderr
                   MU.log stderr, MU::ERR
-                  output << stderr
+                  output_lines << stderr
                 end
               end
             }
 
-            if resp.exitcode == 1 and output.join("\n").match(/Chef Client finished/)
+            if resp.exitcode == 1 and output_lines.join("\n").match(/Chef Client finished/)
               MU.log "resp.exit code 1"
             elsif resp.exitcode != 0
-              raise MU::Cloud::BootstrapTempFail if resp.exitcode == 35 or output.join("\n").match(/REBOOT_SCHEDULED| WARN: Reboot requested:/)
-              raise MU::Groomer::RunError, output.slice(output.length-50, output.length).join("")
+              raise MU::Cloud::BootstrapTempFail if resp.exitcode == 35 or output_lines.join("\n").match(/REBOOT_SCHEDULED| WARN: Reboot requested:|Rebooting server at a recipe's request|Chef::Exceptions::Reboot/)
+              raise MU::Groomer::RunError, output_lines.slice(output_lines.length-50, output_lines.length).join("")
             end
           end
         rescue MU::Cloud::BootstrapTempFail
@@ -397,10 +414,12 @@ module MU
             sleep 30
             retry
           else
+            @server.deploy.sendAdminSlack("Chef run '#{purpose}' failed on `#{@server.mu_name}` :crying_cat_face:", msg: e.message)
             raise MU::Groomer::RunError, "#{@server.mu_name}: Chef run '#{purpose}' failed #{max_retries} times, last error was: #{e.message}"
           end
         rescue Exception => e
-          raise MU::Groomer::RunError, "Caught unexpected #{e.inspect} on #{@server.mu_name} in @groomer.run"
+          @server.deploy.sendAdminSlack("Chef run '#{purpose}' failed on `#{@server.mu_name}` :crying_cat_face:", msg: e.inspect)
+          raise MU::Groomer::RunError, "Caught unexpected #{e.inspect} on #{@server.mu_name} in @groomer.run at #{e.backtrace[0]}"
 
         end
 
@@ -440,20 +459,33 @@ module MU
           end
           guardfile = "/opt/mu_installed_chef"
 
-          ssh = @server.getSSHSession(15)
-          if leave_ours
-            MU.log "Expunging pre-existing Chef install on #{@server.mu_name}, if we didn't create it", MU::NOTICE
-            begin 
-              ssh.exec!(%Q{test -f #{guardfile} || (#{remove_cmd}) ; touch #{guardfile}})
-            rescue IOError => e
-              # TO DO - retry this in a cleaner way
-              MU.log "Got #{e.inspect} while trying to clean up chef, retrying", MU::NOTICE, details: %Q{test -f #{guardfile} || (#{remove_cmd}) ; touch #{guardfile}}
-              ssh = @server.getSSHSession(15)
-              ssh.exec!(%Q{test -f #{guardfile} || (#{remove_cmd}) ; touch #{guardfile}})
+          retries = 0
+          begin
+            ssh = @server.getSSHSession(15)
+            Timeout::timeout(60) {
+              if leave_ours
+                MU.log "Expunging pre-existing Chef install on #{@server.mu_name}, if we didn't create it", MU::NOTICE
+                begin 
+                  ssh.exec!(%Q{test -f #{guardfile} || (#{remove_cmd}) ; touch #{guardfile}})
+                rescue IOError => e
+                  # TO DO - retry this in a cleaner way
+                  MU.log "Got #{e.inspect} while trying to clean up chef, retrying", MU::NOTICE, details: %Q{test -f #{guardfile} || (#{remove_cmd}) ; touch #{guardfile}}
+                  ssh = @server.getSSHSession(15)
+                  ssh.exec!(%Q{test -f #{guardfile} || (#{remove_cmd}) ; touch #{guardfile}})
+                end
+              else
+                MU.log "Expunging pre-existing Chef install on #{@server.mu_name}", MU::NOTICE
+                ssh.exec!(remove_cmd)
+              end
+            }
+          rescue Timeout::Error
+            if retries < 5
+              retries += 1
+              sleep 5
+              retry
+            else
+              raise MuError, "Failed to preClean #{@server.mu_name} after repeated timeouts"
             end
-          else
-            MU.log "Expunging pre-existing Chef install on #{@server.mu_name}", MU::NOTICE
-            ssh.exec!(remove_cmd)
           end
   
           ssh.close
@@ -523,6 +555,7 @@ module MU
       def bootstrap
         self.class.loadChefLib
         stashHostSSLCertSecret
+        splunkVaultInit
         if !@config['cleaned_chef']
           begin
             leave_ours = @config['scrub_groomer'] ? false : true
@@ -654,8 +687,8 @@ retry
           end
         }
         knifeAddToRunList("role[mu-node]")
+        knifeAddToRunList("mu-tools::selinux")
 
-        splunkVaultInit
         grantSecretAccess(@server.mu_name, "windows_credentials") if @server.windows?
         grantSecretAccess(@server.mu_name, "ssl_cert")
 
@@ -669,6 +702,7 @@ retry
           run(purpose: "Base configuration", update_runlist: false, max_retries: 20)
         end
         ::Chef::Knife.run(['node', 'run_list', 'remove', @server.mu_name, "recipe[mu-tools::updates]"], {}) if !@config['skipinitialupdates']
+        ::Chef::Knife.run(['node', 'run_list', 'remove', @server.mu_name, "recipe[mu-tools::selinux]"], {})
 
         # This will deal with Active Directory integration.
         if !@config['active_directory'].nil?
@@ -696,6 +730,11 @@ retry
       # @return [Hash]: The data synchronized.
       def saveDeployData
         self.class.loadChefLib
+        if !haveBootstrapped?
+          MU.log "saveDeployData invoked on #{@server.to_s} before Chef has been bootstrapped!", MU::WARN, details: caller
+          return
+        end
+
         @server.describe(update_cache: true) # Make sure we're fresh
         saveChefMetadata
         begin
@@ -724,10 +763,12 @@ retry
             }
           end
 
-          if chef_node.normal['deployment'] != @server.deploy.deployment
+          if !@server.deploy.deployment.nil? and 
+             (chef_node.normal['deployment'].nil? or 
+               (chef_node.normal['deployment'].to_h <=> @server.deploy.deployment) != 0
+             )
             MU.log "Updating node: #{@server.mu_name} deployment attributes", details: @server.deploy.deployment
             chef_node.normal['deployment'].merge!(@server.deploy.deployment)
-            chef_node.normal['deployment']['ssh_public_key'] = @server.deploy.ssh_public_key
             chef_node.save
           end
           return chef_node['deployment']
@@ -770,6 +811,15 @@ retry
           rescue Net::HTTPServerException
           end
         end
+        MU.log "knife data bag delete #{node}"
+        if !noop
+          knife_cd = ::Chef::Knife::ClientDelete.new(['data', 'bag', 'delete', node])
+          knife_cd.config[:yes] = true
+          begin
+            knife_cd.run
+          rescue Net::HTTPServerException
+          end
+        end
 
         return if nodeonly
 
@@ -778,7 +828,7 @@ retry
         rescue MuNoSuchSecret
         end
         ["crt", "key", "csr"].each { |ext|
-          if File.exists?("#{MU.mySSLDir}/#{node}.#{ext}")
+          if File.exist?("#{MU.mySSLDir}/#{node}.#{ext}")
             MU.log "Removing #{MU.mySSLDir}/#{node}.#{ext}"
             File.unlink("#{MU.mySSLDir}/#{node}.#{ext}") if !noop
           end
@@ -812,6 +862,7 @@ retry
         begin
           chef_node = ::Chef::Node.load(@server.mu_name)
         rescue Net::HTTPServerException
+          @server.deploy.sendAdminSlack("Couldn't load Chef metadata on `#{@server.mu_name}` :crying_cat_face:")
           raise MU::Groomer::RunError, "Couldn't load Chef node #{@server.mu_name}"
         end
 
@@ -821,6 +872,7 @@ retry
 
         chef_node.normal.app = @config['application_cookbook'] if !@config['application_cookbook'].nil?
         chef_node.normal["service_name"] = @config["name"]
+        chef_node.normal["credentials"] = @config["credentials"]
         chef_node.normal["windows_admin_username"] = @config['windows_admin_username']
         chef_node.chef_environment = MU.environment.downcase
         if @server.config['cloud'] == "AWS"
@@ -965,9 +1017,9 @@ retry
         if multiple.size == 0
           multiple = [rl_entry]
         end
-        multiple.each { |rl_entry|
-          if !rl_entry.match(/^role|recipe\[/)
-            rl_entry = "#{type}[#{rl_entry}]"
+        multiple.each { |entry|
+          if !entry.match(/^role|recipe\[/)
+            entry = "#{type}[#{entry}]"
           end
         }
 
@@ -975,27 +1027,27 @@ retry
           role_list = nil
           recipe_list = nil
           missing = false
-          multiple.each { |rl_entry|
-            # Rather than argue about whether to expect a bare rl_entry name or
-            # require rl_entry[rolename], let's just accomodate.
-            if rl_entry.match(/^role\[(.+?)\]/)
-              rl_entry_name = Regexp.last_match(1)
+          multiple.each { |entry|
+            # Rather than argue about whether to expect a bare entry name or
+            # require entry[rolename], let's just accomodate.
+            if entry.match(/^role\[(.+?)\]/)
+              entry_name = Regexp.last_match(1)
               if role_list.nil?
                 query=%Q{#{MU::Groomer::Chef.knife} role list};
                 role_list = %x{#{query}}
               end
-              if !role_list.match(/(^|\n)#{rl_entry_name}($|\n)/)
-                MU.log "Attempting to add non-existent #{rl_entry} to #{@server.mu_name}", MU::WARN
+              if !role_list.match(/(^|\n)#{entry_name}($|\n)/)
+                MU.log "Attempting to add non-existent #{entry} to #{@server.mu_name}", MU::WARN
                 missing = true
               end
-            elsif rl_entry.match(/^recipe\[(.+?)\]/)
-              rl_entry_name = Regexp.last_match(1)
+            elsif entry.match(/^recipe\[(.+?)\]/)
+              entry_name = Regexp.last_match(1)
               if recipe_list.nil?
                 query=%Q{#{MU::Groomer::Chef.knife} recipe list};
                 recipe_list = %x{#{query}}
               end
-              if !recipe_list.match(/(^|\n)#{rl_entry_name}($|\n)/)
-                MU.log "Attempting to add non-existent #{rl_entry} to #{@server.mu_name}", MU::WARN
+              if !recipe_list.match(/(^|\n)#{entry_name}($|\n)/)
+                MU.log "Attempting to add non-existent #{entry} to #{@server.mu_name}", MU::WARN
                 missing = true
               end
             end