cloud-mu 3.1.2 → 3.2.0

data/cookbooks/mu-tools/resources/disk.rb

@@ -15,7 +15,7 @@ action :create do
   devicename = device
 
   if set_gcp_cfg_params
-    devicename.gsub!(/.*?\//, "")
+    devicename= devicename.gsub(/.*?\//, "")
     device = "/dev/disk/by-id/google-"+devicename
   end
 

data/cookbooks/mu-tools/resources/windows_users.rb

@@ -195,48 +195,49 @@ action :config do
   else
     # We want to run ec2config as admin user so Windows userdata executes as admin, however the local admin account doesn't have Logon As a Service right. Domain privileges are set separately
 
-    cookbook_file "c:\\Windows\\SysWOW64\\ntrights.exe" do
-      source "ntrights"
-    end
-    [new_resource.ssh_user, new_resource.ec2config_user].each { |usr|
-			pass = if usr == new_resource.ec2config_user
-				new_resource.ec2config_password
-			elsif usr == new_resource.ssh_user
-				new_resource.ssh_password
-			end
-
-      user usr do
-        password pass
-      end
-
-      group "Administrators" do
-        action :modify
-        members usr
-        append true
-      end
-
-      %w{SeDenyRemoteInteractiveLogonRight SeDenyInteractiveLogonRight SeServiceLogonRight}.each { |privilege|
-        batch "Grant local user #{usr} logon as service right" do
-          code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
-        end
-      }
-
-			# XXX user resource seems not to really be setting password, or is setting			# in such a way that the user is being required to change it. Workaround.
-      powershell_script "Adjust local account params for #{usr}" do
-				code <<-EOH
-					(([adsi]('WinNT://./#{usr}, user')).psbase.invoke('SetPassword', '#{pass}'))
-				EOH
-      end
-
-      if usr == new_resource.ssh_user
-
-        %w{SeCreateTokenPrivilege SeTcbPrivilege SeAssignPrimaryTokenPrivilege}.each { |privilege|
-          batch "Grant local user #{usr} logon as service right" do
-            code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
-          end
-        }
-					
-      end
-    }
+#    cookbook_file "c:\\Windows\\SysWOW64\\ntrights.exe" do
+#      source "ntrights"
+#    end
+#    [new_resource.ssh_user, new_resource.ec2config_user].each { |usr|
+#			pass = if usr == new_resource.ec2config_user
+#				new_resource.ec2config_password
+#			elsif usr == new_resource.ssh_user
+#				new_resource.ssh_password
+#			end
+#
+#      user usr do
+#        password pass
+#        action :modify
+#      end
+#
+#      group "Administrators" do
+#        action :modify
+#        members usr
+#        append true
+#      end
+#
+#      %w{SeDenyRemoteInteractiveLogonRight SeDenyInteractiveLogonRight SeServiceLogonRight}.each { |privilege|
+#        batch "Grant local user #{usr} logon as service right" do
+#          code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
+#        end
+#      }
+#
+#			# XXX user resource seems not to really be setting password, or is setting			# in such a way that the user is being required to change it. Workaround.
+#      powershell_script "Adjust local account params for #{usr}" do
+#				code <<-EOH
+#					(([adsi]('WinNT://./#{usr}, user')).psbase.invoke('SetPassword', '#{pass}'))
+#				EOH
+#      end
+#
+#      if usr == new_resource.ssh_user
+#
+#        %w{SeCreateTokenPrivilege SeTcbPrivilege SeAssignPrimaryTokenPrivilege}.each { |privilege|
+#          batch "Grant local user #{usr} logon as service right" do
+#            code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
+#          end
+#        }
+#					
+#      end
+#    }
   end
 end

data/extras/clean-stock-amis

@@ -18,37 +18,43 @@ require 'json'
 require File.realpath(File.expand_path(File.dirname(__FILE__)+"/../bin/mu-load-config.rb"))
 require 'mu'
 
-credentials = if ARGV[0] and !ARGV[0].empty?
-  ARGV[0]
-else
-  nil
+$opts = Optimist::options do
+  banner <<-EOS
+#{$0} [-c credentials] [-i imagename]
+  EOS
+  opt :credentials, "Use these AWS credentials from mu.yaml instead of the default set", :required => false, :type => :string
+  opt :image, "Purge a specific image, instead of just scrubing old ones", :required => false, :type => :string
 end
 
 filters = [
   {
     name: "owner-id",
-    values: [MU::Cloud::AWS.credToAcct(credentials)]
+    values: [MU::Cloud::AWS.credToAcct($opts[:credentials])]
   }
 ]
 
 
 MU::Cloud::AWS.listRegions.each { | r|
-  images = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_images(
+  images = MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).describe_images(
     filters: filters + [{ "name" => "state", "values" => ["available"]}]
   ).images
   images.each { |ami|
-		if (DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/)
-			snaps = []
-			ami.block_device_mappings.each { |dev|
-				if !dev.ebs.nil?
-					snaps << dev.ebs.snapshot_id
-				end
-			}
-			MU.log "Deregistering #{ami.name} (#{ami.creation_date})", MU::WARN, details: snaps
-			MU::Cloud::AWS.ec2(region: r, credentials: credentials).deregister_image(image_id: ami.image_id)
- 		  snaps.each { |snap_id|
-			  MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_snapshot(snapshot_id: snap_id)
-			}
-		end
+    if ($opts[:image] and ami.name == $opts[:image]) or 
+       ((DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/))
+      snaps = []
+      ami.block_device_mappings.each { |dev|
+        if !dev.ebs.nil?
+          snaps << dev.ebs.snapshot_id
+        end
+      }
+      MU.log "Deregistering #{ami.name}, #{r} (#{ami.creation_date})", MU::WARN, details: snaps
+      begin
+        MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).deregister_image(image_id: ami.image_id)
+      rescue Aws::EC2::Errors::InvalidAMIIDUnavailable
+      end
+        snaps.each { |snap_id|
+          MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
+        }
+    end
   }
 }

data/extras/generate-stock-images

@@ -91,6 +91,7 @@ $opts[:clouds].each { |cloud|
         end
         next if !needed
       end
+      MU.log "Loading "+bok_dir+"/"+cloud+"/"+platform+".yaml"
       conf_engine = MU::Config.new(
         bok_dir+"/"+cloud+"/"+platform+".yaml",
         default_credentials: $opts[(cloud.downcase+"_creds").to_sym]

data/extras/image-generators/AWS/win2k12.yaml

@@ -1,16 +1,21 @@
 ---
   appname: mu
+  vpcs:
+  - name: windowsbuild
   servers: 
-    - 
-      name: win2k12
-      platform: windows
-      size: m4.large
-      scrub_groomer: true
-      run_list:
-      - recipe[mu-tools::updates]
-      - recipe[mu-utility::cleanup_image_helper]
-      create_image:
-        image_then_destroy: true
-        public: true
-        copy_to_regions:
-        - "#ALL"
+  - name: win2k12
+    platform: win2k12
+    vpc:
+      name: windowsbuild
+    size: m4.large
+    scrub_groomer: true
+    groomer: Ansible
+    run_list:
+    - mu-windows
+    ansible_vars:
+      mu_build_image: true
+    create_image:
+      image_then_destroy: true
+      public: true
+      copy_to_regions:
+      - "#ALL"

data/extras/image-generators/AWS/win2k16.yaml

@@ -1,16 +1,21 @@
 ---
   appname: mu
+  vpcs:
+  - name: windowsbuild
   servers: 
-    - 
-      name: win2k16
-      platform: windows
-      size: m4.large
-      scrub_groomer: true
-      run_list:
-      - recipe[mu-tools::updates]
-      - recipe[mu-utility::cleanup_image_helper]
-      create_image:
-        image_then_destroy: true
-        public: true
-        copy_to_regions:
-        - "#ALL"
+  - name: win2k16
+    platform: win2k16
+    vpc:
+      name: windowsbuild
+    size: m4.large
+    scrub_groomer: true
+    groomer: Ansible
+    run_list:
+    - mu-windows
+    ansible_vars:
+      mu_build_image: true
+    create_image:
+      image_then_destroy: true
+      public: true
+      copy_to_regions:
+      - "#ALL"

data/extras/image-generators/AWS/win2k19.yaml

@@ -0,0 +1,21 @@
+---
+  appname: mu
+  vpcs:
+  - name: windowsbuild
+  servers: 
+  - name: win2k19
+    platform: windows
+    vpc:
+      name: windowsbuild
+    size: m4.large
+    scrub_groomer: true
+    groomer: Ansible
+    run_list:
+    - mu-windows
+    ansible_vars:
+      mu_build_image: true
+    create_image:
+      image_then_destroy: true
+      public: true
+      copy_to_regions:
+      - "#ALL"

data/extras/image-generators/Google/centos6.yaml

@@ -16,3 +16,4 @@
     create_image:
       image_then_destroy: true
       public: true
+      family: mu-centos-6

data/extras/image-generators/Google/centos7.yaml

@@ -4,7 +4,6 @@
   - name: centos7
     cloud: Google
     image_id: "centos-cloud/centos-7"
-    platform: centos6
     ssh_user: centos
     size: g1-small
     associate_public_ip: true
@@ -16,3 +15,4 @@
     create_image:
       image_then_destroy: true
       public: true
+      family: mu-centos-7

data/modules/mommacat.ru

@@ -16,7 +16,7 @@ require 'pp'
 require 'base64'
 require 'etc'
 
-home = Etc.getpwuid(Process.uid).dir
+Etc.getpwuid(Process.uid).dir
 
 if !ENV.include?('MU_INSTALLDIR')
   ENV['MU_INSTALLDIR'] = "/opt/mu"
@@ -51,8 +51,8 @@ Signal.trap("URG") do
 end
 
 begin
-  MU::MommaCat.syncMonitoringConfig(false)
-rescue Exception => e
+  MU::Master.syncMonitoringConfig(false)
+rescue StandardError => e
   MU.log e.inspect, MU::ERR, details: e.backtrace
   # ...but don't die!
 end
@@ -64,14 +64,12 @@ Thread.new {
     MU::MommaCat.cleanTerminatedInstances
     MU::Master.cleanExpiredScratchpads if $ENABLE_SCRATCHPAD
     sleep 60
-  rescue Exception => e
+  rescue StandardError => e
     MU.log "Error in cleanTerminatedInstances thread: #{e.inspect}", MU::ERR, details: e.backtrace
     retry
   end while true
 }
 
-required_vars = ["mu_id", "mu_deploy_secret", "mu_resource_name", "mu_resource_type", "mu_instance_id"]
-
 # Use a template to generate a pleasant-looking HTML page for simple messages
 # and errors.
 def genHTMLMessage(title: "", headline: "", msg: "", template: $MU_CFG['html_template'], extra_vars: {})
@@ -193,7 +191,6 @@ def releaseKitten(mu_id)
 end
 
 app = proc do |env|
-  ok = false
   returnval = [
       200,
       {
@@ -318,7 +315,7 @@ app = proc do |env|
         elsif !filter or !path
           returnval = throw404 env['REQUEST_PATH']
         else
-          MU::MommaCat.addInstanceToEtcHosts(path, filter)
+          MU::Master.addInstanceToEtcHosts(path, filter)
           returnval = [
             200,
             {
@@ -356,7 +353,6 @@ app = proc do |env|
 
     elsif !env["rack.input"].nil?
       req = Rack::Utils.parse_nested_query(env["rack.input"].read)
-      ok = true
 
       if req["mu_user"].nil?
         req["mu_user"] = "mu"
@@ -373,7 +369,6 @@ app = proc do |env|
       kittenpile = getKittenPile(req)
       if kittenpile.nil? or kittenpile.original_config.nil? or kittenpile.original_config[req["mu_resource_type"]+"s"].nil?
         returnval = throw500 "Couldn't find config data for #{req["mu_resource_type"]} in deploy_id #{req["mu_id"]}"
-        ok = false
         next
       end
       server_cfg = nil
@@ -385,7 +380,6 @@ app = proc do |env|
       }
       if server_cfg.nil?
         returnval = throw500 "Couldn't find config data for #{req["mu_resource_type"]} name: #{req["mu_resource_name"]} deploy_id: #{req["mu_id"]}"
-        ok = false
         next
       end
 
@@ -426,7 +420,6 @@ MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
           instance.addVolume(params["dev"], params["size"], delete_on_termination: params["delete_on_termination"])
         else
           returnval = throw500 "I don't know how to add a volume for #{instance}"
-          ok = false
         end
       elsif !instance.nil?
         if !req["mu_bootstrap"].nil?
@@ -434,16 +427,13 @@ MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
           returnval[2] = ["Grooming asynchronously, check Momma Cat logs on the master for details."]
         else
           returnval = throw500 "Didn't get 'mu_bootstrap' parameter from instance id '#{req["mu_instance_id"]}'"
-          ok = false
         end
       else
         returnval = throw500 "No such instance id '#{req["mu_instance_id"]}' nor was this an SSL signing request"
-        ok = false
       end
     end
-  rescue Exception => e
+  rescue StandardError => e
     returnval = throw500 "Invalid request: #{e.inspect} (#{req})", e.backtrace
-    ok = false
   ensure
     if !req.nil?
       releaseKitten(req['mu_id'])

data/modules/mu.rb

@@ -79,38 +79,40 @@ class Hash
     }
     return 0 if self == other # that was easy!
     # compare elements and decide who's "bigger" based on their totals?
-    0
+
+    # fine, try some brute force and just hope everything implements to_s
+    self.flatten.map { |e| e.to_s }.join() <=> other.flatten.map { |e| e.to_s }.join()
   end
 
-  # Recursively compare two hashes
-  def diff(with, on = self, level: 0, parents: [])
+  # Recursively compare two Mu Basket of Kittens hashes and report the differences
+  def diff(with, on = self, level: 0, parents: [], report: {}, habitat: nil)
     return if with.nil? and on.nil?
     if with.nil? or on.nil? or with.class != on.class
       return # XXX ...however we're flagging differences
     end
     return if on == with
 
-    tree = ""
-    indentsize = 0
-    parents.each { |p|
-      tree += (" " * indentsize) + p + " => \n"
-      indentsize += 2
-    }
-    indent = (" " * indentsize)
-
     changes = []
+    report ||= {}
     if on.is_a?(Hash)
       on_unique = (on.keys - with.keys)
       with_unique = (with.keys - on.keys)
       shared = (with.keys & on.keys)
       shared.each { |k|
-        diff(with[k], on[k], level: level+1, parents: parents + [k])
+
+        report_data = diff(with[k], on[k], level: level+1, parents: parents + [k], report: report[k], habitat: habitat)
+        if report_data and !report_data.empty?
+          report ||= {}
+          report[k] = report_data
+        end
       }
       on_unique.each { |k|
-        changes << "- ".red+PP.pp({k => on[k] }, '')
+        report[k] = { :action => :removed, :parents => parents, :value => on[k].clone }
+        report[k][:habitat] = habitat if habitat
       }
       with_unique.each { |k|
-        changes << "+ ".green+PP.pp({k => with[k]}, '')
+        report[k] = { :action => :added, :parents => parents, :value => with[k].clone }
+        report[k][:habitat] = habitat if habitat
       }
     elsif on.is_a?(Array)
       return if with == on
@@ -121,34 +123,28 @@ class Hash
       # custom objects which we might find in here so that we can get away with
       # sorting arrays full of weird, non-primitive types.
       done = []
-#      before_a = on.dup
-#      after_a = on.dup.sort
-#      before_b = with.dup
-#      after_b = with.dup.sort
       on.sort.each { |elt|
-        if elt.is_a?(Hash) and elt['name'] or elt['entity']# or elt['cloud_id']
+        if elt.is_a?(Hash) and !MU::MommaCat.getChunkName(elt).first.nil?
+          elt_namestr, elt_location, elt_location_list = MU::MommaCat.getChunkName(elt)
+
           with.sort.each { |other_elt|
-            if (elt['name'] and other_elt['name'] == elt['name']) or
-               (elt['name'].nil? and !elt["id"].nil? and elt["id"] == other_elt["id"]) or
-               (elt['name'].nil? and elt["id"].nil? and
-                !elt["entity"].nil? and !other_elt["entity"].nil? and
-                 (
-                   (elt["entity"]["id"] and elt["entity"]["id"] == other_elt["entity"]["id"]) or
-                   (elt["entity"]["name"] and elt["entity"]["name"] == other_elt["entity"]["name"])
-                 )
+            other_elt_namestr, other_elt_location, other_elt_location_list = MU::MommaCat.getChunkName(other_elt)
+
+            # Case 1: The array element exists in both version of this array
+            if elt_namestr and other_elt_namestr and
+               elt_namestr == other_elt_namestr and
+               (elt_location.nil? or other_elt_location.nil? or
+                elt_location == other_elt_location or
+                !(elt_location_list & other_elt_location_list).empty?
                )
-              break if elt == other_elt
               done << elt
               done << other_elt
-              namestr = if elt['type']
-                "#{elt['type']}[#{elt['name']}]"
-              elsif elt['name']
-                elt['name']
-              elsif elt['entity'] and elt["entity"]["id"]
-                elt['entity']['id']
+              break if elt == other_elt # if they're identical, we're done
+              report_data = diff(other_elt, elt, level: level+1, parents: parents + [elt_namestr], habitat: (elt_location || habitat))
+              if report_data and !report_data.empty?
+                report ||= {}
+                report[elt_namestr] = report_data
               end
-
-              diff(other_elt, elt, level: level+1, parents: parents + [namestr])
               break
             end
           }
@@ -156,43 +152,34 @@ class Hash
       }
       on_unique = (on - with) - done
       with_unique = (with - on) - done
-#    if on_unique.size > 0 or with_unique.size > 0
-#      if before_a != after_a
-#        MU.log "A BEFORE", MU::NOTICE, details: before_a
-#        MU.log "A AFTER", MU::NOTICE, details: after_a
-#      end
-#      if before_b != after_b
-#        MU.log "B BEFORE", MU::NOTICE, details: before_b
-#        MU.log "B AFTER", MU::NOTICE, details: after_b
-#      end
-#    end
+
+      # Case 2: This array entry exists in the old version, but not the new one
       on_unique.each { |e|
-        changes << if e.is_a?(Hash)
-          "- ".red+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
-        else
-          "- ".red+e.to_s
-        end
+        namestr, loc = MU::MommaCat.getChunkName(e)
+
+        report ||= {}
+        report[namestr] = { :action => :removed, :parents => parents, :value => e.clone }
+        report[namestr][:habitat] = loc if loc
       }
+
+      # Case 3: This array entry exists in the new version, but not the old one
       with_unique.each { |e|
-        changes << if e.is_a?(Hash)
-          "+ ".green+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
-        else
-          "+ ".green+e.to_s
-        end
+        namestr, loc = MU::MommaCat.getChunkName(e)
+
+        report ||= {}
+        report[namestr] = { :action => :added, :parents => parents, :value => e.clone }
+        report[namestr][:habitat] = loc if loc
       }
+
+    # A plain old leaf node of data
     else
       if on != with
-        changes << "-".red+" #{on.to_s}"
-        changes << "+".green+" #{with.to_s}"
+        report = { :action => :changed, :parents => parents, :oldvalue => on, :value => with.clone }
+        report[:habitat] = habitat if habitat
       end
     end
 
-    if changes.size > 0
-      puts tree
-      changes.each { |c|
-        puts indent+c
-      }
-    end
+    report.freeze
   end
 
   # Implement a merge! that just updates each hash leaf as needed, not 
@@ -216,8 +203,29 @@ class Hash
 end
 
 ENV['HOME'] = Etc.getpwuid(Process.uid).dir
+module MU
+
+  # For log entries that should only be logged when we're in verbose mode
+  DEBUG = 0.freeze
+  # For ordinary log entries
+  INFO = 1.freeze
+  # For more interesting log entries which are not errors
+  NOTICE = 2.freeze
+  # Log entries for non-fatal errors
+  WARN = 3.freeze
+  # Log entries for non-fatal errors
+  WARNING = 3.freeze
+  # Log entries for fatal errors
+  ERR = 4.freeze
+  # Log entries for fatal errors
+  ERROR = 4.freeze
+  # Log entries that will be held and displayed/emailed at the end of deploy,
+  # cleanup, etc.
+  SUMMARY = 5.freeze
+end
 
 require 'mu/logger'
+
 module MU
 
   # Subclass core thread so we can gracefully handle it when we hit system
@@ -267,6 +275,7 @@ module MU
       end while newguy.nil?
 
       @@mu_global_thread_semaphore.synchronize {
+        MU.dupGlobals(Thread.current.object_id, target_thread: newguy)
         @@mu_global_threads << newguy
       }
 
@@ -276,8 +285,9 @@ module MU
   # Wrapper class for fatal Exceptions. Gives our internals something to
   # inherit that will log an error message appropriately before bubbling up.
   class MuError < StandardError
-    def initialize(message = nil)
-      MU.log message, MU::ERR, details: caller[2] if !message.nil?
+    def initialize(message = nil, silent: false, details: nil)
+      details ||= caller[2]
+      MU.log message, MU::ERR, details: details if !message.nil? and !silent
       if MU.verbosity == MU::Logger::SILENT
         super ""
       else
@@ -289,8 +299,8 @@ module MU
   # Wrapper class for temporary Exceptions. Gives our internals something to
   # inherit that will log a notice message appropriately before bubbling up.
   class MuNonFatal < StandardError
-    def initialize(message = nil)
-      MU.log message, MU::NOTICE if !message.nil?
+    def initialize(message = nil, silent: false)
+      MU.log message, MU::NOTICE if !message.nil? and !silent
       if MU.verbosity == MU::Logger::SILENT
         super ""
       else
@@ -299,6 +309,68 @@ module MU
     end
   end
 
+  # Boilerplate retry block executor, for making cloud API calls which might
+  # fail transiently.
+  #
+  # @param catchme [Array<Exception>]: Exception classes which should be caught and retried
+  # @param wait [Integer]: Number of seconds to wait between retries
+  # @param max [Integer]: Maximum number of retries; if less than 1, will retry indefinitely
+  # @param ignoreme [Array<Exception>]: Exception classes which can be silently treated as success. This will override any +loop_if+ block and return automatically (after invoking +always+, if the latter was specified).
+  # @param on_retry [Proc]: Optional block of code to invoke during retries
+  # @param always [Proc]: Optional block of code to invoke before returning or failing, a bit like +ensure+
+  # @param loop_if [Proc]: Optional block of code to invoke which will cause our block to be rerun until true
+  # @param loop_msg [String]: Message to display every third attempt
+  def self.retrier(catchme = nil, wait: 30, max: 0, ignoreme: [], on_retry: nil, always: nil, loop_if: nil, loop_msg: nil)
+
+    loop_if ||= Proc.new { false }
+
+    retries = 0
+    begin
+      retries += 1
+      loglevel = ((retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
+      log_attempts = retries.to_s
+      log_attempts += (max > 0 ? "/"+max.to_s : "")
+      yield(retries, wait) if block_given?
+      if loop_if.call
+        MU.log loop_msg, loglevel, details: log_attempts if loop_msg
+        sleep wait
+      end
+    rescue StandardError => e
+      if catchme and catchme.include?(e.class)
+        if max > 0 and retries >= max
+          always.call if always and always.is_a?(Proc)
+          if ignoreme.include?(e.class)
+            return
+          else
+            raise e
+          end
+        end
+
+        if on_retry and on_retry.is_a?(Proc)
+          on_retry.call(e)
+        end
+
+        if retries == max-1
+          MU.log e.message, MU::WARN, details: caller
+          sleep wait # wait extra on the final attempt
+        else
+          MU.log e.message, loglevel, details: log_attempts
+        end
+
+        sleep wait
+        retry
+      elsif ignoreme and ignoreme.include?(e.class)
+        always.call if always and always.is_a?(Proc)
+        return
+      else
+        always.call if always and always.is_a?(Proc)
+        raise e
+      end
+    end while loop_if.call and (max < 1 or retries < max)
+
+    always.call if always and always.is_a?(Proc)
+  end
+
   if !ENV.has_key?("MU_LIBDIR") and ENV.has_key?("MU_INSTALLDIR")
     ENV['MU_LIBDIR'] = ENV['MU_INSTALLDIR']+"/lib"
   else
@@ -349,6 +421,13 @@ module MU
 
     if Gem.paths and Gem.paths.home and File.dirname(__FILE__).match(/^#{Gem.paths.home}/)
       @in_gem = true
+    elsif Gem.paths and Gem.paths.path and !Gem.paths.path.empty?
+      Gem.paths.path.each { |p|
+        if File.dirname(__FILE__).match(/^#{Regexp.quote(p)}/)
+          @in_gem = true
+        end
+      }
+      @in_gem = false if !defined? @in_gem
     else
       @in_gem = false
     end
@@ -391,20 +470,20 @@ module MU
   @@global_var_semaphore = Mutex.new
 
   # Set one of our global per-thread variables.
-  def self.setVar(name, value)
+  def self.setVar(name, value, target_thread: Thread.current)
     @@global_var_semaphore.synchronize {
-      @@globals[Thread.current.object_id] ||= Hash.new
-      @@globals[Thread.current.object_id][name] ||= Hash.new
-      @@globals[Thread.current.object_id][name] = value
+      @@globals[target_thread.object_id] ||= Hash.new
+      @@globals[target_thread.object_id][name] ||= Hash.new
+      @@globals[target_thread.object_id][name] = value
     }
   end
 
   # Copy the set of global variables in use by another thread, typically our
   # parent thread.
-  def self.dupGlobals(parent_thread_id)
+  def self.dupGlobals(parent_thread_id, target_thread: Thread.current)
     @@globals[parent_thread_id] ||= {}
     @@globals[parent_thread_id].each_pair { |name, value|
-      setVar(name, value)
+      setVar(name, value, target_thread: target_thread)
     }
   end
 
@@ -532,9 +611,10 @@ module MU
   end
 
   # Shortcut to invoke {MU::Logger#log}
-  def self.log(msg, level = MU::INFO, details: nil, html: false, verbosity: nil, color: true)
+  def self.log(msg, level = MU::INFO, shorthand_details = nil, details: nil, html: false, verbosity: nil, color: true)
     return if (level == MU::DEBUG and verbosity and verbosity <= MU::Logger::LOUD)
     return if verbosity and verbosity == MU::Logger::SILENT
+    details ||= shorthand_details
 
     if (level == MU::ERR or
         level == MU::WARN or
@@ -553,25 +633,6 @@ module MU
     @@logger.log(msg, level, details: details, html: html, verbosity: verbosity, color: color)
   end
 
-  # For log entries that should only be logged when we're in verbose mode
-  DEBUG = 0.freeze
-  # For ordinary log entries
-  INFO = 1.freeze
-  # For more interesting log entries which are not errors
-  NOTICE = 2.freeze
-  # Log entries for non-fatal errors
-  WARN = 3.freeze
-  # Log entries for non-fatal errors
-  WARNING = 3.freeze
-  # Log entries for fatal errors
-  ERR = 4.freeze
-  # Log entries for fatal errors
-  ERROR = 4.freeze
-  # Log entries that will be held and displayed/emailed at the end of deploy,
-  # cleanup, etc.
-  SUMMARY = 5.freeze
-
-
   autoload :Cleanup, 'mu/cleanup'
   autoload :Deploy, 'mu/deploy'
   autoload :MommaCat, 'mu/mommacat'
@@ -585,7 +646,7 @@ module MU
     new_cfg = $MU_CFG.dup
     examples = {}
     MU::Cloud.supportedClouds.each { |cloud|
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+      cloudclass = MU::Cloud.cloudClass(cloud)
       begin
         if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
           cfg_blob = cloudclass.hosted_config
@@ -642,7 +703,7 @@ module MU
      !$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
     @@mu_public_addr = $MU_CFG['public_address']
     if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/)
-      hostname = IO.readlines("/etc/hostname")[0].gsub /\n/, ''
+      hostname = IO.readlines("/etc/hostname")[0].gsub(/\n/, '')
 
       hostlines = File.open('/etc/hosts').grep(/.*#{hostname}.*/)
       if hostlines and !hostlines.empty?
@@ -741,11 +802,7 @@ module MU
   # @param groomer [String]: The grooming agent to load.
   # @return [Class]: The class object implementing this groomer agent
   def self.loadGroomer(groomer)
-    if !File.size?(MU.myRoot+"/modules/mu/groomers/#{groomer.downcase}.rb")
-      raise MuError, "Requested to use unsupported grooming agent #{groomer}"
-    end
-    require "mu/groomers/#{groomer.downcase}"
-    return Object.const_get("MU").const_get("Groomer").const_get(groomer)
+    MU::Groomer.loadGroomer(groomer)
   end
 
   @@myRegion_var = nil
@@ -899,8 +956,7 @@ module MU
 
   @@myCloudDescriptor = nil
   if MU.myCloud
-    svrclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud).const_get("Server")
-    found = svrclass.find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
+    found = MU::Cloud.resourceClass(MU.myCloud, "Server").find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
 #    found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
     if !found.nil? and found.size == 1
       @@myCloudDescriptor = found.values.first
@@ -913,8 +969,7 @@ module MU
   def self.myVPCObj
     return nil if MU.myCloud.nil?
     return @@myVPCObj_var if @@myVPCObj_var
-    cloudclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud)
-    @@myVPCObj_var ||= cloudclass.myVPCObj
+    @@myVPCObj_var ||= MU::Cloud.cloudClass(MU.myCloud).myVPCObj
     @@myVPCObj_var
   end
 
@@ -1012,15 +1067,15 @@ module MU
 
   # Generate a random password which will satisfy the complexity requirements of stock Amazon Windows AMIs.
   # return [String]: A password string.
-  def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 25)
+  def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 50)
     # We have dopey complexity requirements, be stringent here.
     # I'll be nice and not condense this into one elegant-but-unreadable regular expression
     attempts = 0
     safe_metachars = Regexp.escape(safe_pattern)
     begin
       if attempts > retries
-        MU.log "Failed to generate an adequate Windows password after #{attempts}", MU::ERR
-        raise MuError, "Failed to generate an adequate Windows password after #{attempts}"
+        MU.log "Failed to generate an adequate Windows password after #{attempts} attempts", MU::ERR
+        raise MuError, "Failed to generate an adequate Windows password after #{attempts} attempts"
       end
       winpass = Password.random(14..16)
       attempts += 1
@@ -1039,10 +1094,9 @@ module MU
 
     clouds = platform.nil? ? MU::Cloud.supportedClouds : [platform]
     clouds.each { |cloud|
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-      bucketname = cloudclass.adminBucketName(credentials)
+      bucketname = MU::Cloud.cloudClass(cloud).adminBucketName(credentials)
       begin
-        if platform or (cloudclass.hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
+        if platform or (MU::Cloud.cloudClass(cloud).hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
           return bucketname
         end
       end