cloud-mu 1.9.0.pre.beta

data/bin/mu-node-manage

@@ -0,0 +1,683 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'optimist'
+require 'json'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+$opts = Optimist::options do
+  banner <<-EOS
+Usage:
+#{$0} [-c] [-w] [-l] [-d] [-a] [-e <environment>] [-p <platform>] [-m <mode>] [-o <chefopts>] [-x <command>] [ deploy_id|node_name [ ... ] ]
+  EOS
+  opt :concurrent, "Max number of processes to run concurrently when invoking Chef or MommaCat on multiple nodes.", :require => false, :default => 10, :type => :integer
+  opt :list, "Perform no action, but instead return a list of matching hosts. Default behavior with no other flags.", :require => false, :default => false, :type => :boolean
+  opt :deploys, "Operate on matching deploy IDs instead of node names.", :require => false, :default => false, :type => :boolean
+  opt :all, "Operate on all nodes/deploys. Use with caution.", :require => false, :default => false, :type => :boolean
+  opt :platform, "Operate exclusively on one nodes of a particular operating system. Can be used in conjunction with -a or -d. Valid platforms: linux, windows", :require => false, :type => :string
+  opt :environment, "Operate exclusively on one nodes with a particular environment (e.g. dev, prod). Can be used in conjunction with -a or -d.", :require => false, :type => :string
+  opt :override_chef_runlist, "An alternate runlist to pass to Chef, in chefrun mode.", :require => false, :type => :string
+  opt :xecute, "Run a shell command on matching nodes. Overrides --mode and suppresses some informational output in favor of scriptability.", :require => false, :type => :string
+  opt :mode, "Action to perform on matching nodes. Valid actions: groom, chefrun, awsmeta, vaults, certs, chefupgrade", :require => false, :default => "chefrun", :type => :string
+  opt :verbose, "Show output from Chef runs, etc", :require => false, :default => false, :type => :boolean
+  opt :winrm, "Force WinRM connection. Disable SSH fallback", :require => false, :default => false, :type => :boolean
+  opt :info, "List a particular node attribute", :require => false, :default => 'nodename', :type => :string
+end
+
+MU.setLogging(MU::Logger::LOUD) if $opts[:verbose]
+
+if !["groom", "chefrun", "vaults", "userdata", "awsmeta", "certs", "chefupgrade"].include?($opts[:mode])
+  Optimist::die(:mode, "--mode must be one of: groom, chefrun, awsmeta, vaults, certs, chefupgrade")
+end
+if $opts[:platform] and !["linux", "windows"].include?($opts[:platform])
+  Optimist::die(:platform, "--platform must be one of: linux, windows")
+end
+if !$opts[:xecute] and !$opts[:override_chef_runlist] and !$opts[:mode_given]
+  $opts[:list] = true
+end
+if $opts[:override_chef_runlist] and !$opts[:mode_given]
+  $opts[:mode_given] = true
+end
+if ARGV.empty? and !$opts[:all] and !$opts[:platform] and !$opts[:environment] and !$opts[:list]
+  Optimist::educate
+  exit 1
+end
+
+Thread.abort_on_exception = true
+master_pid = Process.pid
+$children = {}
+signals = Signal.list
+signals.keys.each { |sig|
+  # Ruby 2.3 doesn't want to trap these
+  next if ["ILL", "FPE", "KILL", "BUS", "SEGV", "STOP", "VTALRM"].include?(sig)
+  Signal.trap(signals[sig]) do
+    if Process.pid == master_pid
+      $children.each_pair { |pid, node|
+        if ["INT", "TERM", "EXIT", "ABRT"].include?(sig)
+          Process.kill("KILL", pid) # aka --dammit
+        else
+          begin
+            Process.kill(sig, pid)
+          rescue Errno::ESRCH
+          end
+        end
+      }
+      if ["INT", "TERM", "EXIT"].include?(sig)
+        Process.waitall
+      end
+    end
+  end
+}
+
+# Run through our filters so we can pass flat lists into our methods that
+# actually do things.
+avail_deploys = MU::MommaCat.listDeploys
+do_deploys = []
+do_nodes = []
+ok = true
+if $opts[:all]
+  do_deploys = avail_deploys
+else
+  if $opts[:deploys] and !$opts[:all]
+    ARGV.each { |arg|
+      matched = avail_deploys.select { |deploy| deploy.match(/#{Regexp.quote(arg)}/i) }
+      if matched.size == 0
+        MU.log "Deploy pattern '#{arg}' doesn't appear to match anything", MU::ERR
+      end
+      do_deploys = do_deploys.concat(matched).uniq
+    }
+  else
+    do_nodes = ARGV
+    do_deploys = []
+    if do_nodes.size > 0
+      # Just load the deploys we need
+      do_nodes.each { |node|
+        if node.match(/^(.*?-[^\-]+?-\d{10}-[A-Z]{2})-.*/)
+          do_deploys << node.sub(/^(.*?-[^\-]+?-\d{10}-[A-Z]{2})-.*/, '\1')
+        end
+      }
+      do_deploys.uniq!
+    end
+    do_deploys = avail_deploys if do_deploys.size == 0
+  end
+end
+
+avail_nodes = []
+@avail_node_attributes = []
+
+do_deploys.each { |muid|
+  mommacat = MU::MommaCat.new(muid, skip_resource_objects: true)
+  mommacat.listNodes.each_pair { |nodename, server|
+    next if server.nil? or server['conf'].nil?
+    id = server['instance_id']
+    server['conf']["platform"] = "linux" if !server['conf'].has_key?("platform") or %w{centos centos6 centos7 ubuntu ubuntu14 rhel rhel7 rhel71 linux amazon}.include?(server['conf']["platform"])
+    server['conf']["platform"] = "windows" if %w{win2k12r2 win2k12 win2k8 win2k8r2}.include?(server['conf']["platform"])
+    next if !$opts[:platform].nil? and server['conf']["platform"] != $opts[:platform]
+    next if !$opts[:environment].nil? and MU.environment.upcase != $opts[:environment].upcase
+    avail_nodes << nodename
+    @avail_node_attributes << server
+  }
+}
+
+if do_nodes.size > 0
+  matching = avail_nodes.select { |node|
+    matched = false
+    do_nodes.each { |pattern|
+      if node.match(/#{Regexp.quote(pattern)}/i)
+        matched = true
+        break
+      end
+    }
+    matched
+  }
+  do_nodes = matching
+else
+  do_nodes = avail_nodes
+end
+do_nodes.sort!
+
+#do_nodes.sort!{ |x,y| (x[$opts[:info]] <=> y[$opts[:info]]) }
+
+#puts node_attributes[0]
+
+if $opts[:list]
+  if $opts[:info].eql? 'nodename'
+    puts do_nodes
+  else
+    do_nodes.each do |node|
+      @avail_node_attributes.each do |attr|
+        if attr['nodename'].eql? node
+          puts "#{attr['nodename']}: #{attr[$opts[:info]]}"
+        end
+      end
+    end
+  end
+  exit
+
+end
+
+exit 1 if !ok
+
+
+def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
+  badnodes = []
+  count = 0
+  deploys.each { |muid|
+    mommacat = MU::MommaCat.new(muid)
+    next if mommacat.kittens.nil? or mommacat.kittens['servers'].nil?
+    mommacat.kittens['servers'].each_pair { |nodeclass, servers|
+      servers.each_pair { |mu_name, server|
+        next if nodes.size > 0 and !nodes.include?(mu_name)
+        count = count + 1
+        child = Process.fork {
+          begin
+            type = "server"
+            type = "server_pool" if server.config.has_key?("basis")
+            if vaults_only
+              next if !server.config.has_key?("vault_access")
+              server.config["vault_access"].each { |v|
+                MU::Groomer::Chef.grantSecretAccess(mu_name, v['vault'], v['item'])
+              }
+            else
+              mommacat.groomNode(server.cloud_id, nodeclass, type, mu_name: mu_name)
+            end
+          rescue Exception => e
+            MU.log e.inspect, MU::ERR, details: e.backtrace
+            exit 1
+          end
+        }
+        $children[child] = mu_name
+      }
+      while $children.size >= $opts[:concurrent]-1
+        child = Process.wait
+        if !$?.success?
+          badnodes << $children[child]
+        end
+        $children.delete(child)
+      end
+    }
+  }
+  Process.waitall.each { |child|
+    if !child[1].success?
+      badnodes << $children[child[0]]
+    end
+  }
+
+  if badnodes.size > 0
+    MU.log "Not all Momma Cat runs exited cleanly", MU::WARN, details: badnodes
+  end
+end
+
+def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_output: $opts[:verbose], noop: false, chefrun: false, chef_runlist: nil)
+  badnodes = []
+  count = 0
+  deploys.each { |muid|
+    mommacat = MU::MommaCat.new(muid)
+    mommacat.listNodes.each_pair { |nodename, server|
+      next if server['conf'].nil?
+      server['conf']["platform"] = "linux" if !server['conf'].has_key?("platform")
+      next if nodes.size > 0 and !nodes.include?(nodename)
+
+      count = count + 1
+      child = Process.fork {
+        done = false
+        begin
+          serverobj = mommacat.findLitterMate(type: "server", mu_name: nodename)
+
+          # Generate the command if attemting a chef run
+          if chefrun
+            cmd = serverobj.windows? ? "powershell -Command chef-client" : "chef-client || sudo chef-client"
+            cmd += " -o '#{chef_runlist}'" if chef_runlist
+          end
+
+          MU.log "Running '#{cmd}' on #{nodename} (##{count})" if !print_output
+
+          # Set Variables to catch the output and exit code of the execution
+          output = nil
+          exitcode = -1
+
+          # Determine which protocols to attempt
+          if serverobj.windows?
+            attempt_winrm = true
+            if $opts[:winrm]
+              attempt_ssh = false
+            else
+              attempt_ssh = true
+            end
+          else
+            attempt_winrm = false
+            attempt_ssh = true
+          end
+
+          # Attempt WinRM Connection, and Fall back to SSH
+          if attempt_winrm
+            exec_protocol = 'WinRM'
+            
+            # Attempt to make a connection and exec the command
+            resp = nil
+            begin
+              shell = serverobj.getWinRMSession(0, timeout: 10, winrm_retries: 1)
+              resp = shell.run(cmd)
+            rescue MU::MuError => e
+            end
+
+            if resp
+              # WINRM CONNECTION AND EXECUTION SUCCESS
+              output = resp.stdout if resp.stdout
+              exitcode = resp.exitcode if resp.exitcode
+
+              if exitcode.eql? 0
+                attempt_ssh = false
+              else
+                puts resp.stderr if resp.stderr
+                puts output
+              end
+            end
+
+            if exitcode != 0
+              if attempt_ssh
+                MU.log "#{nodename} WinRM exec failed, trying SSH", MU::NOTICE
+              else
+                MU.log "#{nodename} WinRM exec failed, NOT trying SSH", MU::WARN
+              end
+            end
+          end
+
+          if attempt_ssh
+            exec_protocol = 'SSH'
+
+            # this should use getSSHSession, for the sake of symmetry
+            output = `ssh -q #{nodename} "#{cmd}" 2>&1 < /dev/null`
+            exitcode = $?.exitstatus
+          end
+
+          if exitcode != 0
+
+            if output
+
+              if serverobj.windows? and output.match(/NoMethodError: unknown property or method: `ConnectServer'/)
+                MU.log "#{nodename} encountered transient Windows/Chef ConnectServer error, retrying", MU::WARN
+              elsif print_output
+                done = true
+                puts "#{nodename} - #{output}" if output.match(/[^\s]/)
+                MU.log "#{nodename} did not exit cleanly", MU::WARN
+              else
+                done = true
+                MU.log "#{nodename} did not exit cleanly", MU::WARN, details: output.slice(-2000, 2000)
+              end
+
+            else
+              done = true
+              MU.log "#{nodename} did not exit cleanly", MU::WARN
+            end
+
+            exit exitcode if done
+
+          else
+            MU.log "#{nodename} complete via #{exec_protocol}"
+            done = true
+          end
+
+        end until done
+
+        puts "#{nodename} - #{output}" if print_output and output.match(/[^\s]/)
+
+      }
+      $children[child] = nodename
+      while $children.size >= $opts[:concurrent] - 1
+        child = Process.wait
+        if !$?.success?
+          badnodes << $children[child]
+        end
+        $children.delete(child)
+      end
+    }
+  }
+  Process.waitall.each { |child|
+    if !child[1].success?
+      badnodes << $children[child[0]]
+    end
+  }
+
+  if badnodes.size > 0
+    cmd = "Chef" if $opts[:mode] == "chefrun"
+    if !print_output
+      MU.log "Not all `#{cmd}` runs exited cleanly", MU::WARN, details: badnodes
+    else
+      MU.log "Not all `#{cmd}` runs exited cleanly", MU::WARN
+    end
+  end
+end
+
+def updateAWSMetaData(deploys = MU::MommaCat.listDeploys, nodes = [])
+  deploys.each { |muid|
+    mommacat = MU::MommaCat.new(muid)
+
+    if mommacat.original_config.nil?
+      MU.log "Failed to locate original config data for #{muid}", MU::WARN
+      next
+    end
+
+    # Clean up the userdata of matching Autoscale groups by replacing their
+    # Launch Configurations with new ones,
+    if mommacat.original_config.has_key?("server_pools")
+      mommacat.original_config['server_pools'].each { |server|
+        svr_class = server['name']
+        server["platform"] = "linux" if !server.has_key?("platform")
+
+        pool_name = mommacat.getResourceName(svr_class)
+        if nodes.size > 0
+          matched = false
+          nodes.each { |n|
+            if n.match(/^#{Regexp.quote(pool_name)}-[a-z0-9]{3}$/i)
+              matched = true
+            end
+          }
+          next if !matched
+        end
+
+        MU::Cloud::AWS::Server.createIAMProfile(pool_name, base_profile: server['iam_role'], extra_policies: server['iam_policies'])
+
+        resp = MU::Cloud::AWS.autoscale.describe_auto_scaling_groups(
+            auto_scaling_group_names: [pool_name]
+        )
+
+        if resp.nil?
+          MU.log "Failed to locate any Autoscale Groups named #{pool_name}", MU::WARN
+          next
+        end
+        resp.auto_scaling_groups.each { |asg|
+          launch = MU::Cloud::AWS.autoscale.describe_launch_configurations(
+            launch_configuration_names: [asg.launch_configuration_name]
+          ).launch_configurations.first
+
+          olduserdata = Base64.decode64(launch.user_data)
+
+          userdata = MU::Cloud::AWS::Server.fetchUserdata(
+            platform: server["platform"],
+            template_variables: {
+              "deployKey" => Base64.urlsafe_encode64(mommacat.public_key),
+              "deploySSHKey" => mommacat.ssh_public_key,
+              "muID" => muid,
+              "muUser" => MU.chef_user,
+              "publicIP" => MU.mu_public_ip,
+              "resourceName" => svr_class,
+              "windowsAdminName" => server['windows_admin_username'],
+              "skipApplyUpdates" => server['skipinitialupdates'],
+              "resourceType" => "server_pool"
+            },
+            custom_append: server['userdata_script']
+          )
+
+          # Figure out which devices are embedded in the AMI already.
+          image = MU::Cloud::AWS.ec2.describe_images(image_ids: [server["basis"]["launch_config"]["ami_id"]]).images.first
+
+          if image.nil?
+            MU.log "#{server["basis"]["launch_config"]["ami_id"]} does not exist, skipping launch config #{asg.launch_configuration_name}", MU::ERR
+            next
+          end
+
+          ext_disks = {}
+          if !image.block_device_mappings.nil?
+            image.block_device_mappings.each { |disk|
+              if !disk.device_name.nil? and !disk.device_name.empty? and !disk.ebs.nil? and !disk.ebs.empty?
+                ext_disks[disk.device_name] = MU.structToHash(disk.ebs)
+                if ext_disks[disk.device_name].has_key?(:snapshot_id)
+                  ext_disks[disk.device_name].delete(:encrypted)
+                end
+              end
+            }
+          end
+
+          storage = []
+          if !server["basis"]["launch_config"]["storage"].nil?
+            server["basis"]["launch_config"]["storage"].each { |vol|
+              if ext_disks.has_key?(vol["device"])
+                if ext_disks[vol["device"]].has_key?(:snapshot_id)
+                  vol.delete("encrypted")
+                end
+              end
+              mapping, cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
+              storage << mapping
+            }
+          end
+
+          storage.concat(MU::Cloud::AWS::Server.ephemeral_mappings)
+          if userdata != olduserdata or
+              launch.image_id != server["basis"]["launch_config"]["ami_id"] or
+              launch.ebs_optimized != server["basis"]["launch_config"]["ebs_optimized"] or
+              launch.instance_type != server["basis"]["launch_config"]["size"] or
+              launch.instance_monitoring.enabled != server["basis"]["launch_config"]["monitoring"]
+#							launch.block_device_mappings != storage
+#							XXX block device comparison isn't this simple
+            need_update = true
+          end
+          next if !need_update
+
+          # Put our Autoscale group onto a temporary launch config
+          begin
+            MU::Cloud::AWS.autoscale.create_launch_configuration(
+              launch_configuration_name: pool_name+"-TMP",
+              user_data: Base64.encode64(userdata),
+              image_id: server["basis"]["launch_config"]["ami_id"],
+              key_name: launch.key_name,
+              security_groups: launch.security_groups,
+              instance_type: server["basis"]["launch_config"]["size"],
+              block_device_mappings: storage,
+              instance_monitoring: {:enabled => server["basis"]["launch_config"]["monitoring"]},
+              iam_instance_profile: launch.iam_instance_profile,
+              ebs_optimized: server["basis"]["launch_config"]["ebs_optimized"],
+              associate_public_ip_address: launch.associate_public_ip_address
+            )
+          rescue ::Aws::AutoScaling::Errors::ValidationError => e
+            if e.message.match(/Member must have length less than or equal to (\d+)/)
+              MU.log "Userdata script too long updating #{pool_name} Launch Config (#{Base64.encode64(userdata).size.to_s}/#{Regexp.last_match[1]} bytes)", MU::ERR
+            else
+              MU.log "Error updating #{pool_name} Launch Config", MU::ERR, details: e.message
+            end
+            next
+          end
+
+          MU::Cloud::AWS.autoscale.update_auto_scaling_group(
+            auto_scaling_group_name: pool_name,
+            launch_configuration_name: pool_name+"-TMP"
+          )
+
+          # ...now back to an identical one with the "real" name
+          MU::Cloud::AWS.autoscale.delete_launch_configuration(
+              launch_configuration_name: pool_name
+          )
+          MU::Cloud::AWS.autoscale.create_launch_configuration(
+              launch_configuration_name: pool_name,
+              user_data: Base64.encode64(userdata),
+              image_id: server["basis"]["launch_config"]["ami_id"],
+              key_name: launch.key_name,
+              security_groups: launch.security_groups,
+              instance_type: server["basis"]["launch_config"]["size"],
+              block_device_mappings: storage,
+              instance_monitoring: {:enabled => server["basis"]["launch_config"]["monitoring"]},
+              iam_instance_profile: launch.iam_instance_profile,
+              ebs_optimized: server["basis"]["launch_config"]["ebs_optimized"],
+              associate_public_ip_address: launch.associate_public_ip_address
+          )
+          MU::Cloud::AWS.autoscale.update_auto_scaling_group(
+            auto_scaling_group_name: pool_name,
+            launch_configuration_name: pool_name
+          )
+          MU::Cloud::AWS.autoscale.delete_launch_configuration(
+            launch_configuration_name: pool_name+"-TMP"
+          )
+
+          MU.log "Launch Configuration #{asg.launch_configuration_name} replaced"
+        }
+      }
+    end
+
+    # Update the userdata of live nodes. They must be in the Stopped state for
+    # us to do so.
+    mommacat.listNodes.each_pair { |nodename, server|
+      if server['conf'].nil?
+        MU.log "Failed to find config data for server #{nodename}", MU::WARN
+        next
+      end
+      id = server['cloud_id']
+      id = server['instance_id'] if id.nil?
+      desc = MU::Cloud::AWS.ec2(server['region']).describe_instances(instance_ids: [id]).reservations.first.instances.first
+
+      server['conf']["platform"] = "linux" if !server['conf'].has_key?("platform")
+      next if nodes.size > 0 and !nodes.include?(nodename)
+
+      rolename, cfm_role_name, cfm_prof_name, arn = MU::Cloud::AWS::Server.createIAMProfile(nodename, base_profile: server["conf"]['iam_role'], extra_policies: server["conf"]['iam_policies'])
+      MU::Cloud::AWS::Server.addStdPoliciesToIAMProfile(rolename)
+
+      mytype = "server"
+      mytype = "server_pool" if server['conf'].has_key?("basis") or server['conf']['#TYPENAME'] == "ServerPool" or server['conf']["#MU_CLASS"] == "MU::Cloud::AWS::ServerPool"
+      olduserdata = Base64.decode64(MU::Cloud::AWS.ec2(server['region']).describe_instance_attribute(
+                                instance_id: id,
+                                attribute: "userData"
+                              ).user_data.value)
+
+      userdata = MU::Cloud::AWS::Server.fetchUserdata(
+          platform: server['conf']["platform"],
+          template_variables: {
+              "deployKey" => Base64.urlsafe_encode64(mommacat.public_key),
+              "deploySSHKey" => mommacat.ssh_public_key,
+              "muID" => muid,
+              "muUser" => MU.chef_user,
+              "publicIP" => MU.mu_public_ip,
+              "resourceName" => server['conf']['name'],
+              "windowsAdminName" => server['conf']['windows_admin_username'],
+              "skipApplyUpdates" => server['conf']['skipinitialupdates'],
+              "resourceType" => mytype
+          },
+          custom_append: server['userdata_script']
+      )
+
+      if userdata == olduserdata
+        MU.log "#{nodename} has up-to-date userdata, skipping", MU::DEBUG
+        next
+      end
+
+      if desc.state.name != "stopped"
+        MU.log "#{nodename} needs a userdata update, but is not in Stopped state", MU::NOTICE
+        if mytype == "server_pool"
+          pool_name = mommacat.getResourceName(server['conf']['name'])
+          MU.log "Note: Be sure to pause Autoscaling for this group before stopping this instance, e.g. with: aws autoscaling suspend-processes --auto-scaling-group-name #{pool_name}", MU::WARN
+        end
+        next
+      end
+
+      MU.log "Updating #{nodename} userdata (#{server["conf"]["platform"]})"
+      begin
+        MU::Cloud::AWS.ec2(server['region']).modify_instance_attribute(
+          instance_id: id,
+          attribute: "userData",
+          value: Base64.encode64(userdata)
+        )
+      rescue ::Aws::EC2::Errors::InvalidParameterValue => e
+        if e.message.match(/User data is limited to (\d+)/)
+          MU.log "Userdata script too long updating #{nodename} (#{userdata.size.to_s}/#{Regexp.last_match[1]} bytes)", MU::ERR
+        else
+          MU.log "Error replacing userData on #{nodename}", MU::ERR, details: e.message
+        end
+      end
+    }
+  }
+end
+
+def sslCerts(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
+  badnodes = []
+  count = 0
+  deploys.each { |muid|
+    mommacat = MU::MommaCat.new(muid)
+    mommacat.listNodes.each_pair { |nodename, server|
+      next if server['conf'].nil?
+      server['conf']["platform"] = "linux" if !server['conf'].has_key?("platform")
+      next if nodes.size > 0 and !nodes.include?(nodename)
+      if server['conf'].nil?
+        MU.log "Failed to find config data for server #{nodename}", MU::WARN
+        next
+      end
+
+      server_obj = mommacat.findLitterMate(type: "server", mu_name: nodename)
+      mommacat.nodeSSLCerts(server_obj)
+    }
+  }
+end
+
+def chefUpgrade(deploys = MU::MommaCat.listDeploys, nodes = [])
+  badnodes = []
+  deploys.each { |muid|
+    mommacat = MU::MommaCat.new(muid)
+    mommacat.listNodes.each_pair { |nodename, server|
+      next if server['conf'].nil?
+      server['conf']["platform"] = "linux" if !server['conf'].has_key?("platform")
+      next if nodes.size > 0 and !nodes.include?(nodename)
+      if server['conf'].nil?
+        MU.log "Failed to find config data for server #{nodename}", MU::WARN
+        next
+      end
+
+      child = Process.fork {
+        server_obj = mommacat.findLitterMate(type: "server", mu_name: nodename)
+        begin
+          server_obj.groomer.reinstall
+        rescue Exception
+        end
+      }
+      $children[child] = nodename
+      while $children.size >= $opts[:concurrent]-1
+        child = Process.wait
+        if !$?.success?
+          badnodes << $children[child]
+        end
+        $children.delete(child)
+      end
+    }
+  }
+
+  Process.waitall.each { |child|
+    if !child[1].success?
+      badnodes << $children[child[0]]
+    end
+  }
+
+  if badnodes.size > 0
+    MU.log "Not all Chef upgrades exited cleanly", MU::WARN, details: badnodes
+  end
+end
+
+if $opts[:xecute]
+  runCommand(do_deploys, do_nodes, $opts[:xecute], print_output: true)
+elsif $opts[:mode] == "certs"
+  sslCerts(do_deploys, do_nodes)
+elsif $opts[:mode] == "groom"
+  reGroom(do_deploys, do_nodes)
+elsif $opts[:mode] == "vaults"
+  reGroom(do_deploys, do_nodes, vaults_only: true)
+elsif $opts[:mode] == "chefupgrade"
+  chefUpgrade(do_deploys, do_nodes)
+elsif $opts[:mode] == "chefrun"
+  print_output = $opts[:verbose] || do_nodes.size == 1
+  if $opts[:override_chef_runlist]
+    runCommand(do_deploys, do_nodes, chef_runlist: $opts[:override_chef_runlist], chefrun: true, print_output: print_output)
+  else
+    runCommand(do_deploys, do_nodes, chefrun: true, print_output: print_output)
+  end
+elsif $opts[:mode] == "userdata" or $opts[:mode] == "awsmeta"
+# Need Google equiv and to select nodes correctly based on what cloud they're in
+  updateAWSMetaData(do_deploys, do_nodes)
+end