cloud-mu 3.1.3 → 3.1.4

data/modules/mu/defaults/Google.yaml

@@ -1,6 +1,6 @@
 ---
-centos6: &centos6 centos-cloud/centos-6
-centos7: &centos7 centos-cloud/centos-7
+centos6: &centos6 egt-labs-admin/mu-centos-6
+centos7: &centos7 egt-labs-admin/mu-centos-7
 rhel71: &rhel71 rhel-cloud/rhel-7
 rhel6: &rhel6 rhel-cloud/rhel-6
 debian10: &debian10 debian-cloud/debian-10

data/modules/mu/deploy.rb

@@ -126,7 +126,7 @@ module MU
           seedsize = 1 + (retries/10).abs
           seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
           deploy_id = @appname.upcase + "-" + @environment.upcase + "-" + @timestamp + "-" + seed.upcase
-        end while MU::MommaCat.deploy_exists?(deploy_id) or seed == "mu" or seed[0] == seed[1]
+        end while MU::MommaCat.deploy_exists?(deploy_id) or seed == "mu"
         MU.setVar("deploy_id", deploy_id)
         MU.setVar("appname", @appname.upcase)
         MU.setVar("environment", @environment.upcase)
@@ -140,7 +140,7 @@ module MU
 
       @fromName = MU.muCfg['mu_admin_email']
 
-      MU::Cloud.resource_types.each { |cloudclass, data|
+      MU::Cloud.resource_types.values.each { |data|
         if !@main_config[data[:cfg_plural]].nil? and @main_config[data[:cfg_plural]].size > 0
           @main_config[data[:cfg_plural]].each { |resource|
             if force_cloudformation
@@ -154,7 +154,7 @@ module MU
               end
             end
           }
-          shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(data[:cfg_plural])
+          _shortclass, _cfg_name, _cfg_plural, classname = MU::Cloud.getResourceNames(data[:cfg_plural])
           @main_config[data[:cfg_plural]].each { |resource|
             resource["#MU_CLOUDCLASS"] = classname
           }
@@ -274,7 +274,7 @@ module MU
           MU.dupGlobals(parent_thread_id)
           Thread.current.thread_variable_set("name", "mu_create_container")
 #          Thread.abort_on_exception = false
-          MU::Cloud.resource_types.each { |cloudclass, data|
+          MU::Cloud.resource_types.values.each { |data|
             if !@main_config[data[:cfg_plural]].nil? and
                 @main_config[data[:cfg_plural]].size > 0 and
                 data[:instance].include?(:create)
@@ -288,7 +288,7 @@ module MU
           MU.dupGlobals(parent_thread_id)
           Thread.current.thread_variable_set("name", "mu_groom_container")
 #          Thread.abort_on_exception = false
-          MU::Cloud.resource_types.each { |cloudclass, data|
+          MU::Cloud.resource_types.values.each { |data|
             if !@main_config[data[:cfg_plural]].nil? and
                 @main_config[data[:cfg_plural]].size > 0 and
                 data[:instance].include?(:groom)
@@ -311,7 +311,9 @@ module MU
 
         @mommacat.save!
 
-      rescue Exception => e
+      rescue StandardError => e
+        MU.log e.class.name, MU::ERR, details: caller
+
         @my_threads.each do |t|
           if t.object_id != Thread.current.object_id and
              t.thread_variable_get("name") != "main_thread" and
@@ -338,6 +340,8 @@ module MU
             @nocleanup = true # so we don't run this again later
           end
         end
+
+
         @reraise_thread.raise MuError, e.inspect, e.backtrace if @reraise_thread
         Thread.current.exit
       ensure
@@ -596,7 +600,6 @@ MESSAGE_END
       return if services.nil?
 
       parent_thread_id = Thread.current.object_id
-      parent_thread = Thread.current
       services.uniq!
       services.each do |service|
         begin
@@ -639,14 +642,14 @@ MESSAGE_END
               else
                 raise e
               end
-            rescue Exception => e
+            rescue StandardError => e
               MU::MommaCat.unlockAll
               @main_thread.raise MuError, "Error instantiating object from #{myservice["#MU_CLOUDCLASS"]} (#{e.inspect})", e.backtrace
               raise e
             end
             begin
               run_this_method = myservice['#MUOBJECT'].method(mode)
-            rescue Exception => e
+            rescue StandardError => e
               MU::MommaCat.unlockAll
               @main_thread.raise MuError, "Error invoking #{myservice["#MU_CLOUDCLASS"]}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
               raise e
@@ -703,7 +706,7 @@ MESSAGE_END
               @my_threads.reject! { |thr| !thr.alive? }
               sleep 10+Random.rand(20)
               retry
-            rescue Exception => e
+            rescue StandardError => e
               MU.log e.inspect, MU::ERR, details: e.backtrace if @verbosity != MU::Logger::SILENT
               MU::MommaCat.unlockAll
               Thread.list.each do |t|

data/modules/mu/groomer.rb

@@ -121,7 +121,7 @@ module MU
           else
             retval = @groomer_obj.method(method).call
           end
-        rescue Exception => e
+        rescue StandardError => e
         pp e.backtrace
           raise MU::Groomer::RunError, e.message, e.backtrace
         end

data/modules/mu/groomers/ansible.rb

@@ -174,7 +174,6 @@ module MU
           raise MuNoSuchSecret, "No such vault #{vault}"
         end
 
-        data = nil
         if item
           itempath = dir+"/"+item
           if !File.exist?(itempath)
@@ -191,7 +190,7 @@ module MU
 
       # see {MU::Groomer::Ansible.deleteSecret}
       def deleteSecret(vault: nil, item: nil)
-        self.class.deleteSecret(vault: vault, item: nil)
+        self.class.deleteSecret(vault: vault, item: item)
       end
 
       # Invoke the Ansible client on the node at the other end of a provided SSH
@@ -207,22 +206,60 @@ module MU
 
         ssh_user = @server.config['ssh_user'] || "root"
 
-        cmd = %Q{cd #{@ansible_path} && #{@ansible_execs}/ansible-playbook -i hosts #{@server.config['name']}.yml --limit=#{@server.mu_name} --vault-password-file #{pwfile} --timeout=30 --vault-password-file #{@ansible_path}/.vault_pw -u #{ssh_user}}
+        if update_runlist
+          bootstrap
+        end
+
+        tmpfile = nil
+        playbook = if override_runlist and !override_runlist.empty?
+          play = {
+            "hosts" => @server.config['name']
+          }
+          play["become"] = "yes" if @server.config['ssh_user'] != "root"
+          play["roles"] = override_runlist if @server.config['run_list'] and !@server.config['run_list'].empty?
+          play["vars"] = @server.config['ansible_vars'] if @server.config['ansible_vars']
+
+          tmpfile = Tempfile.new("#{@server.config['name']}-override-runlist.yml")
+          tmpfile.puts [play].to_yaml
+          tmpfile.close
+          tmpfile.path
+        else
+          "#{@server.config['name']}.yml"
+        end
+
+        cmd = %Q{cd #{@ansible_path} && echo "#{purpose}" && #{@ansible_execs}/ansible-playbook -i hosts #{playbook} --limit=#{@server.mu_name} --vault-password-file #{pwfile} --timeout=30 --vault-password-file #{@ansible_path}/.vault_pw -u #{ssh_user}}
 
         retries = 0
         begin
           MU.log cmd
-          raise MU::Groomer::RunError, "Failed Ansible command: #{cmd}" if !system(cmd)
-        rescue MU::Groomer::RunError => e
+          Timeout::timeout(timeout) {
+            if output
+              system("#{cmd}")
+            else
+              %x{#{cmd} 2>&1}
+            end
+
+            if $?.exitstatus != 0
+              raise MU::Groomer::RunError, "Failed Ansible command: #{cmd}"
+            end
+          }
+        rescue Timeout::Error, MU::Groomer::RunError => e
           if retries < max_retries
+            if reboot_first_fail and e.class.name == "MU::Groomer::RunError"
+              @server.reboot
+              reboot_first_fail = false
+            end
             sleep 30
             retries += 1
             MU.log "Failed Ansible run, will retry (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE, details: cmd
             retry
           else
+            tmpfile.unlink if tmpfile
             raise MuError, "Failed Ansible command: #{cmd}"
           end
         end
+
+        tmpfile.unlink if tmpfile
       end
 
       # This is a stub; since Ansible is effectively agentless, this operation
@@ -265,7 +302,7 @@ module MU
       # Synchronize the deployment structure managed by {MU::MommaCat} into some Ansible variables, so that nodes can access this metadata.
       # @return [Hash]: The data synchronized.
       def saveDeployData
-        @server.describe(update_cache: true) # Make sure we're fresh
+        @server.describe
 
         allvars = {
           "mu_deployment" => MU::Config.stripConfig(@server.deploy.deployment),
@@ -316,14 +353,15 @@ module MU
 
       # Expunge Ansible resources associated with a node.
       # @param node [String]: The Mu name of the node in question.
-      # @param vaults_to_clean [Array<Hash>]: Some vaults to expunge
+      # @param _vaults_to_clean [Array<Hash>]: Dummy argument, part of this method's interface but not used by the Ansible layer
       # @param noop [Boolean]: Skip actual deletion, just state what we'd do
-      # @param nodeonly [Boolean]: Just delete the node and its keys, but leave other artifacts
-      def self.cleanup(node, vaults_to_clean = [], noop = false, nodeonly: false)
+      def self.cleanup(node, _vaults_to_clean = [], noop = false)
         deploy = MU::MommaCat.new(MU.deploy_id)
         inventory = Inventory.new(deploy)
-        ansible_path = deploy.deploy_dir+"/ansible"
-        inventory.remove(node)
+#        ansible_path = deploy.deploy_dir+"/ansible"
+        if !noop
+          inventory.remove(node)
+        end
       end
 
       # List the Ansible vaults, if any, owned by the specified Mu user
@@ -344,8 +382,7 @@ module MU
       # the results to +STDOUT+.
       # @param name [String]: The variable name to use for the string's YAML key
       # @param string [String]: The string to encrypt
-      # @param for_user [String]: Encrypt using the Vault password of the specified Mu user
-      def self.encryptString(name, string, for_user = nil)
+      def self.encryptString(name, string)
         pwfile = vaultPasswordFile
         cmd = %Q{#{ansibleExecDir}/ansible-vault}
         if !system(cmd, "encrypt_string", string, "--name", name, "--vault-password-file", pwfile)
@@ -353,8 +390,8 @@ module MU
         end
       end
 
-      private
-
+      # Hunt down and return a path for Ansible executables
+      # @return [String]
       def self.ansibleExecDir
         path = nil
         if File.exist?(BINDIR+"/ansible-playbook")
@@ -376,8 +413,12 @@ module MU
         path
       end
 
-      # Get the +.vault_pw+ file for the appropriate user. If it doesn't exist,
-      # generate one.
+      # Get path to the +.vault_pw+ file for the appropriate user. If it
+      # doesn't exist, generate it. 
+      #
+      # @param for_user [String]:
+      # @param pwfile [String]
+      # @return [String]
       def self.vaultPasswordFile(for_user = nil, pwfile: nil)
         pwfile ||= secret_dir(for_user)+"/.vault_pw"
         @@pwfile_semaphore.synchronize {
@@ -392,11 +433,8 @@ module MU
       end
 
       # Figure out where our main stash of secrets is, and make sure it exists
-      def secret_dir
-        MU::Groomer::Ansible.secret_dir(@mu_user)
-      end
-
-      # Figure out where our main stash of secrets is, and make sure it exists
+      # @param user [String]:
+      # @return [String]
       def self.secret_dir(user = MU.mu_user)
         path = MU.dataDir(user) + "/ansible-secrets"
         Dir.mkdir(path, 0755) if !Dir.exist?(path)
@@ -404,6 +442,13 @@ module MU
         path
       end
 
+      private
+
+      # Figure out where our main stash of secrets is, and make sure it exists
+      def secret_dir
+        MU::Groomer::Ansible.secret_dir(@mu_user)
+      end
+
       # Make an effort to distinguish an Ansible role from other sorts of
       # artifacts, since 'roles' is an awfully generic name for a directory.
       # Short of a full, slow syntax check, this is the best we're liable to do.
@@ -543,7 +588,7 @@ module MU
         def haveNode?(name)
           lock
           read
-          @inv.each_pair { |group, nodes|
+          @inv.values.each { |nodes|
             if nodes.include?(name)
               unlock
               return true
@@ -574,7 +619,7 @@ module MU
         def remove(name)
           lock
           read
-          @inv.each_pair { |group, nodes|
+          @inv.each_pair { |_group, nodes|
             nodes.delete(name)
           }
           save!

data/modules/mu/groomers/chef.rb

@@ -184,13 +184,13 @@ module MU
         if !item.nil?
           begin
             loaded = ::ChefVault::Item.load(vault, item)
-          rescue ::ChefVault::Exceptions::KeysNotFound => e
+          rescue ::ChefVault::Exceptions::KeysNotFound
             raise MuNoSuchSecret, "Can't load the Chef Vault #{vault}:#{item}. Does it exist? Chef user: #{MU.chef_user}"
           end
         else
           # If we didn't ask for a particular item, list what we have.
           begin
-            loaded = ::Chef::DataBag.load(vault).keys.select { |k, v| !k.match(/_keys$/) }
+            loaded = ::Chef::DataBag.load(vault).keys.select { |k| !k.match(/_keys$/) }
           rescue Net::HTTPServerException
             raise MuNoSuchSecret, "Failed to retrieve Vault #{vault}"
           end
@@ -258,7 +258,6 @@ module MU
           knifeAddToRunList(multiple: @config['run_list'])
         end
 
-        pending_reboot_count = 0
         chef_node = ::Chef::Node.load(@server.mu_name)
         if !@config['application_attributes'].nil?
           MU.log "Setting node:#{@server.mu_name} application_attributes", MU::DEBUG, details: @config['application_attributes']
@@ -300,7 +299,7 @@ module MU
               cmd = "#{upgrade_cmd} chef-client --color || echo #{error_signal}"
             end
             Timeout::timeout(timeout) {
-              retval = ssh.exec!(cmd) { |ch, stream, data|
+              ssh.exec!(cmd) { |_ch, _stream, data|
                 extra_logfile = if Dir.exist?(@server.deploy.deploy_dir)
                   File.open(@server.deploy.deploy_dir+"/log", "a")
                 end
@@ -380,7 +379,7 @@ module MU
             sleep 30
           end
           retry
-        rescue RuntimeError, SystemCallError, Timeout::Error, SocketError, Errno::ECONNRESET, IOError, Net::SSH::Exception, MU::Groomer::RunError, WinRM::WinRMError, MU::MuError => e
+        rescue SystemExit, Timeout::Error, MU::Cloud::BootstrapTempFail, Net::HTTPServerException, HTTPClient::ConnectTimeoutError, WinRM::WinRMError, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, Net::SSH::Exception, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::EPIPE, SocketError, IOError => e
           begin
             ssh.close if !ssh.nil?
           rescue Net::SSH::Exception, IOError => e
@@ -390,6 +389,8 @@ module MU
               MU.log "ssh session to #{@server.mu_name} was closed unexpectedly, waiting before trying again", MU::NOTICE
             end
             sleep 10
+          rescue StandardError => e
+            MU.log "Error I don't recognize closing ssh tunnel", MU::WARN, details: e.inspect
           end
           if e.instance_of?(MU::Groomer::RunError) and retries == 0 and max_retries > 1 and purpose != "Base Windows configuration"
             MU.log "Got a run error, will attempt to install/update Chef Client on next attempt", MU::NOTICE
@@ -404,7 +405,7 @@ module MU
               begin
                 preClean(true) # drop any Chef install that's not ours
                 @server.reboot # try gently rebooting the thing
-              rescue Exception => e # it's ok to fail here (and to ignore failure)
+              rescue StandardError => e # it's ok to fail here (and to ignore failure)
                 MU.log "preclean err #{e.inspect}", MU::ERR
               end
               reboot_first_fail = false
@@ -429,7 +430,7 @@ module MU
             @server.deploy.sendAdminSlack("Chef run '#{purpose}' failed on `#{@server.mu_name}` :crying_cat_face:", msg: e.message)
             raise MU::Groomer::RunError, "#{@server.mu_name}: Chef run '#{purpose}' failed #{max_retries} times, last error was: #{e.message}"
           end
-        rescue Exception => e
+        rescue StandardError => e
           @server.deploy.sendAdminSlack("Chef run '#{purpose}' failed on `#{@server.mu_name}` :crying_cat_face:", msg: e.inspect)
           raise MU::Groomer::RunError, "Caught unexpected #{e.inspect} on #{@server.mu_name} in @groomer.run at #{e.backtrace[0]}"
 
@@ -443,8 +444,8 @@ module MU
       def splunkVaultInit
         self.class.loadChefLib
         begin
-          loaded = ::ChefVault::Item.load("splunk", "admin_user")
-        rescue ::ChefVault::Exceptions::KeysNotFound => e
+          ::ChefVault::Item.load("splunk", "admin_user")
+        rescue ::ChefVault::Exceptions::KeysNotFound
           pw = Password.pronounceable(12..14)
           creds = {
             "username" => "admin",
@@ -550,7 +551,7 @@ module MU
             winrm = @server.getWinRMSession(1, 30, winrm_retries: 2)
             pp winrm.run(cmd)
             return
-          rescue Net::SSH::Disconnect, SystemCallError, Timeout::Error, Errno::ECONNRESET, Errno::EHOSTUNREACH, Net::SSH::Proxy::ConnectError, SocketError, Net::SSH::Disconnect, Net::SSH::AuthenticationFailed, IOError, Net::HTTPServerException, SystemExit, Errno::ECONNREFUSED, Errno::EPIPE, WinRM::WinRMError, HTTPClient::ConnectTimeoutError, RuntimeError, MU::Cloud::BootstrapTempFail, MU::MuError => e
+          rescue SystemExit, Timeout::Error, MU::Cloud::BootstrapTempFail, MU::MuError, Net::HTTPServerException, HTTPClient::ConnectTimeoutError, WinRM::WinRMError, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, Net::SSH::Exception, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::EPIPE, SocketError, IOError
             MU.log "WinRM failure attempting Chef upgrade on #{@server.mu_name}, will fall back to ssh", MU::WARN
             cmd = %Q{powershell.exe -inputformat none -noprofile "#{cmd}"}
           end
@@ -558,7 +559,7 @@ module MU
 
         MU.log "Attempting Chef upgrade via ssh on #{@server.mu_name}", MU::NOTICE, details: cmd
         ssh = @server.getSSHSession(1)
-        retval = ssh.exec!(cmd) { |ch, stream, data|
+        ssh.exec!(cmd) { |_ch, _stream, data|
           puts data
         }
       end
@@ -580,7 +581,7 @@ module MU
           @config['cleaned_chef'] = true
         end
 
-        nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_addr, ssh_user, ssh_key_name = @server.getSSHConfig
+        _nat_ssh_key, _nat_ssh_user, _nat_ssh_host, canonical_addr, ssh_user, ssh_key_name = @server.getSSHConfig
 
         MU.log "Bootstrapping #{@server.mu_name} (#{canonical_addr}) with knife"
 
@@ -593,11 +594,12 @@ module MU
           json_attribs['skipinitialupdates'] = @config['skipinitialupdates']
         end
 
-        if !@config['vault_access'].nil?
-          vault_access = @config['vault_access']
-        else
-          vault_access = []
-        end
+# XXX this seems to break Knife Bootstrap
+#        vault_access = if !@config['vault_access'].nil?
+#          @config['vault_access']
+#        else
+#          []
+#        end
 
         @server.windows? ? max_retries = 25 : max_retries = 10
         @server.windows? ? timeout = 1800 : timeout = 300
@@ -658,7 +660,7 @@ module MU
           }
           # throws Net::HTTPServerException if we haven't really bootstrapped
           ::Chef::Node.load(@server.mu_name)
-        rescue Net::SSH::Disconnect, SystemCallError, Timeout::Error, Errno::ECONNRESET, Errno::EHOSTUNREACH, Net::SSH::Proxy::ConnectError, SocketError, Net::SSH::Disconnect, Net::SSH::AuthenticationFailed, IOError, Net::HTTPServerException, SystemExit, Errno::ECONNREFUSED, Errno::EPIPE, WinRM::WinRMError, HTTPClient::ConnectTimeoutError, RuntimeError, MU::Cloud::BootstrapTempFail, Net::SSH::Exception, Net::SSH::ConnectionTimeout => e
+        rescue SystemExit, Timeout::Error, MU::Cloud::BootstrapTempFail, Net::HTTPServerException, HTTPClient::ConnectTimeoutError, WinRM::WinRMError, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, Net::SSH::Exception, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::EPIPE, SocketError, IOError => e
           if retries < max_retries
             retries += 1
             # Bad Chef installs are possible culprits of bootstrap failures, so
@@ -671,7 +673,7 @@ module MU
                !@config['forced_preclean']
               begin
                 preClean(false) # it's ok for this to fail
-              rescue Exception => e
+              rescue StandardError => e
               end
               MU::Groomer::Chef.cleanup(@server.mu_name, nodeonly: true)
               @config['forced_preclean'] = true
@@ -683,7 +685,7 @@ module MU
           else
             raise MuError, "#{@server.mu_name}: Knife Bootstrap failed too many times with #{e.inspect}"
           end
-        rescue Exception => e
+        rescue StandardError => e
 MU.log e.inspect, MU::ERR, details: e.backtrace
 sleep 10*retries
 retry
@@ -700,7 +702,7 @@ retry
           end
         }
         knifeAddToRunList("role[mu-node]")
-        knifeAddToRunList("mu-tools::selinux")
+        knifeAddToRunList("recipe[mu-tools::selinux]")
 
         grantSecretAccess(@server.mu_name, "windows_credentials") if @server.windows?
         grantSecretAccess(@server.mu_name, "ssl_cert")
@@ -748,7 +750,7 @@ retry
           return
         end
 
-        @server.describe(update_cache: true) # Make sure we're fresh
+        @server.describe
         saveChefMetadata
         begin
           chef_node = ::Chef::Node.load(@server.mu_name)
@@ -785,7 +787,7 @@ retry
             chef_node.save
           end
           return chef_node['deployment']
-        rescue Net::HTTPServerException => e
+        rescue Net::HTTPServerException
           MU.log "Attempted to save deployment to Chef node #{@server.mu_name} before it was bootstrapped.", MU::DEBUG
         end
       end
@@ -804,7 +806,7 @@ retry
             MU.log "knife vault remove #{vault['vault']} #{vault['item']} --search name:#{node}", MU::NOTICE
             begin
               ::Chef::Knife.run(['vault', 'remove', vault['vault'], vault['item'], "--search", "name:#{node}"]) if !noop
-            rescue Exception => e
+            rescue StandardError => e
               MU.log "Error removing vault access for #{node} from #{vault['vault']} #{vault['item']}", MU::ERR, details: e.inspect
             end
             MU::MommaCat.unlock("vault-#{vault['vault']}")
@@ -858,7 +860,7 @@ retry
                   ::Chef::Knife.run(['vault', 'refresh', vault['vault'], vault['item']])
                 end
               end
-            rescue JSON::ParserError => e
+            rescue JSON::ParserError
               MU.log "Error parsing JSON from data bag #{vault['vault']} #{vault['item']}_keys, skipping vault client cleanse", MU::WARN
             end
           end
@@ -887,18 +889,34 @@ retry
         MU.log "Granting #{host} access to #{vault} #{item}"
         begin
           ::Chef::Knife.run(['vault', 'update', vault, item, "--search", "name:#{host}"])
-        rescue Exception => e
+        rescue StandardError => e
           MU.log e.inspect, MU::ERR, details: caller
         end
         MU::MommaCat.unlock("vault-#{vault}", true)
       end
 
+      # Execute a +knife+ command, and return its exit status and output
+      # @param cmd [String]: The knife subcommand to run, such as +vault list+
+      # @param showoutput [String]: Print the results to stdout
+      # @return [Array<Integer,String>]
+      def self.knifeCmd(cmd, showoutput = false)
+        MU.log "knife #{cmd}", MU::NOTICE if showoutput
+        output = `#{MU::Groomer::Chef.knife} #{cmd}`
+        exitstatus = $?.exitstatus
+
+        if showoutput
+          puts output
+          puts "Exit status: #{exitstatus}"
+        end
+        return [exitstatus, output]
+      end
+
       private
 
       # Save common Mu attributes to this node's Chef node structure.
       def saveChefMetadata
         self.class.loadChefLib
-        nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_addr, ssh_user, ssh_key_name = @server.getSSHConfig
+        @server.getSSHConfig # why though
         MU.log "Saving #{@server.mu_name} Chef artifacts"
 
         begin
@@ -998,7 +1016,7 @@ retry
           deploy = MU::MommaCat.getLitter(MU.deploy_id, use_cache: false)
           @config['dependencies'].each{ |dep|
             if dep['type'] == "database" && deploy.deployment.has_key?("databases") && deploy.deployment["databases"].has_key?(dep['name'])
-                deploy.deployment["databases"][dep['name']].each { |name, database|
+                deploy.deployment["databases"][dep['name']].values.each { |database|
                 grantSecretAccess(database['vault_name'], database['vault_item']) if database.has_key?("vault_name") && database.has_key?("vault_item")
               }
             end
@@ -1019,18 +1037,6 @@ retry
         @secrets_granted["#{vault}:#{item}"] = item
       end
 
-      def self.knifeCmd(cmd, showoutput = false)
-        MU.log "knife #{cmd}", MU::NOTICE if showoutput
-        output = `#{MU::Groomer::Chef.knife} #{cmd}`
-        exitstatus = $?.exitstatus
-
-        if showoutput
-          puts output
-          puts "Exit status: #{exitstatus}"
-        end
-        return [exitstatus, output]
-      end
-
       def knifeCmd(cmd, showoutput = false)
         self.class.knifeCmd(cmd, showoutput)
       end
@@ -1063,9 +1069,11 @@ retry
         if multiple.size == 0
           multiple = [rl_entry]
         end
-        multiple.each { |entry|
+        multiple.map! { |entry|
           if !entry.match(/^role|recipe\[/)
-            entry = "#{type}[#{entry}]"
+            "#{type}[#{entry}]"
+          else
+            entry
           end
         }
 
@@ -1110,8 +1118,8 @@ retry
           MU.log("Adding #{rl_string} to Chef run_list of #{@server.mu_name}")
           MU.log("Running #{query}", MU::DEBUG)
           output=%x{#{query}}
-            # XXX rescue Exception is bad style
-        rescue Exception => e
+            # XXX rescue StandardError is bad style
+        rescue StandardError => e
           raise MuError, "FAIL: #{MU::Groomer::Chef.knife} node run_list add #{@server.mu_name} \"#{rl_string}\": #{e.message} (output was #{output})"
         end
       end