cloud-mu 2.1.0beta → 3.0.0beta

data/install/installer

@@ -2,7 +2,7 @@
 
 BOLD=`tput bold`
 NORM=`tput sgr0`
-CHEF_CLIENT_VERSION="14.11.21"
+CHEF_CLIENT_VERSION="14.13.11"
 if [ "$MU_BRANCH" == "" ];then
   MU_BRANCH="master"
   mydir="`dirname $0`"

data/install/mu-master.yaml

@@ -0,0 +1,55 @@
+---
+appname: mu
+parameters:
+- name: cloud
+  default: <%= MU.myCloud || "AWS" %>
+  valid_values:
+<% MU::Cloud.availableClouds.each { |c| %>
+  - <%= c %>
+<% } %>
+- name: public
+  default: true
+- name: name
+  default: mu-master
+scrub_mu_isms: true
+servers: 
+- name: <%= name %>
+  groomer: Ansible
+  run_list:
+  - mu-installer
+  platform: centos7
+  cloud: <%= cloud %>
+<% if cloud == "AWS" %>
+  size: t2.medium
+<% elsif cloud == "Azure" %>
+  size: Standard_DS1_v2
+<% elsif cloud == "Google" %>
+  size: n1-standard-1
+<% end %>
+  vpc:
+    name: <%= name %>-vpc
+<% if public == "true" %>
+    subnet_pref: public
+  static_ip:
+    assign_ip: true
+  associate_public_ip: true
+<% else %>
+    subnet_pref: private
+<% end %>
+<% if cloud == "AWS" %>
+  canned_iam_policies: 
+  - AdministratorAccess
+<% elsif cloud == "Azure" %>
+  roles:
+  - Owner
+<% elsif cloud == "Google" %>
+  roles:
+  - role:
+      id: roles/owner
+<% end %>
+vpcs:
+- name: <%= name %>-vpc
+  cloud: <%= cloud %>
+<% if public %>
+  create_bastion: false
+<% end %>

data/modules/mommacat.ru

@@ -32,7 +32,12 @@ $LOAD_PATH << "#{$MUDIR}/modules"
 require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
 require 'mu'
 
-MU::Groomer::Chef.loadChefLib # pre-cache this so we don't take a hit on a user-interactive need
+begin
+  MU::Groomer::Chef.loadChefLib # pre-cache this so we don't take a hit on a user-interactive need
+  $ENABLE_SCRATCHPAD = true
+rescue LoadError
+  MU.log "Chef libraries not available, disabling Scratchpad", MU::WARN
+end
 #MU.setLogging($opts[:verbose], $opts[:web])
 if MU.myCloud == "AWS"
   MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
@@ -57,7 +62,7 @@ Thread.new {
   MU.dupGlobals(parent_thread_id)
   begin
     MU::MommaCat.cleanTerminatedInstances
-    MU::Master.cleanExpiredScratchpads
+    MU::Master.cleanExpiredScratchpads if $ENABLE_SCRATCHPAD
     sleep 60
   rescue Exception => e
     MU.log "Error in cleanTerminatedInstances thread: #{e.inspect}", MU::ERR, details: e.backtrace
@@ -199,6 +204,17 @@ app = proc do |env|
   ]
   begin
     if !env.nil? and !env['REQUEST_PATH'].nil? and env['REQUEST_PATH'].match(/^\/scratchpad/)
+      if !$ENABLE_SCRATCHPAD
+        msg = "Scratchpad disabled in non-Chef Mu installations"
+        return [
+          504,
+          {
+            'Content-Type' => 'text/html',
+            'Content-Length' => msg.length.to_s
+          },
+          [msg]
+        ]
+      end
       itemname = env['REQUEST_PATH'].sub(/^\/scratchpad\//, "")
       begin
         if itemname.sub!(/\/secret$/, "")
@@ -257,7 +273,7 @@ app = proc do |env|
             [page]
           ]
         end
-      rescue MU::Groomer::Chef::MuNoSuchSecret
+      rescue MU::Groomer::MuNoSuchSecret
         page = nil
         if $MU_CFG.has_key?('scratchpad') and
            $MU_CFG['scratchpad'].has_key?("template_path") and
@@ -287,6 +303,7 @@ app = proc do |env|
         ]
       end
     elsif !env.nil? and !env['REQUEST_PATH'].nil? and env['REQUEST_PATH'].match(/^\/rest\//)
+
       action, filter, path = env['REQUEST_PATH'].sub(/^\/rest\/?/, "").split(/\//, 3)
       # Don't give away the store. This can't be public until we can
       # authenticate and access-control properly.
@@ -295,7 +312,23 @@ app = proc do |env|
         next
       end
 
-      if action == "deploy"
+      if action == "hosts_add"
+        if Process.uid != 0
+          returnval = throw500 "Service not available"
+        elsif !filter or !path
+          returnval = throw404 env['REQUEST_PATH']
+        else
+          MU::MommaCat.addInstanceToEtcHosts(path, filter)
+          returnval = [
+            200,
+            {
+              'Content-Type' => 'text/plain',
+              'Content-Length' => 2
+            },
+            ["ok"]
+          ]
+        end
+      elsif action == "deploy"
         returnval = throw404 env['REQUEST_PATH'] if !filter
         MU.log "Loading deploy data for #{filter} #{path}"
         kittenpile = MU::MommaCat.getLitter(filter)
@@ -313,9 +346,9 @@ app = proc do |env|
           200,
           {
             'Content-Type' => 'text/plain',
-            'Content-Length' => MU.adminBucketName.length.to_s
+            'Content-Length' => MU.adminBucketName(filter, credentials: path).length.to_s
           },
-          [MU.adminBucketName]
+          [MU.adminBucketName(filter, credentials: path)]
         ]
       else
         returnval = throw404 env['REQUEST_PATH']
@@ -389,7 +422,8 @@ app = proc do |env|
         if instance.respond_to?(:addVolume)
 # XXX make sure we handle mangled input safely
           params = JSON.parse(Base64.decode64(req["add_volume"]))
-          instance.addVolume(params["dev"], params["size"])
+MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
+          instance.addVolume(params["dev"], params["size"], delete_on_termination: params["delete_on_termination"])
         else
           returnval = throw500 "I don't know how to add a volume for #{instance}"
           ok = false

data/modules/mu.rb

@@ -36,20 +36,252 @@ class Object
   end
 end
 
+# Mu extensions to Ruby's {Hash} type for internal Mu use
+class Hash
+
+  # Strip extraneous fields out of a {MU::Config} hash to make it suitable for
+  # shorthand printing, such as with <tt>mu-adopt --diff</tt>
+  def self.bok_minimize(o)
+    if o.is_a?(Hash)
+      newhash = o.reject { |k, v|
+        !v.is_a?(Array) and !v.is_a?(Hash) and !["name", "id", "cloud_id"].include?(k)
+      }
+#      newhash.delete("cloud_id") if newhash["name"] or newhash["id"]
+      newhash.each_pair { |k, v|
+        newhash[k] = bok_minimize(v)
+      }
+      newhash.reject! { |_k, v| v.nil? or v.empty? }
+      newhash = newhash.values.first if newhash.size == 1
+      return newhash
+    elsif o.is_a?(Array)
+      newarray = []
+      o.each { |v|
+        newvalue = bok_minimize(v)
+        newarray << newvalue if !newvalue.nil? and !newvalue.empty?
+      }
+      newarray = newarray.first if newarray.size == 1
+      return newarray
+    end
+
+    o
+  end
+
+  # A comparison function for sorting arrays of hashes
+  def <=>(other)
+    return 1 if other.nil? or self.size > other.size
+    return -1 if other.size > self.size
+    # Sort any array children we have
+    self.each_pair { |k, v|
+      self[k] = v.sort if v.is_a?(Array)
+    }
+    other.each_pair { |k, v|
+      other[k] = v.sort if v.is_a?(Array)
+    }
+    return 0 if self == other # that was easy!
+    # compare elements and decide who's "bigger" based on their totals?
+    0
+  end
+
+  # Recursively compare two hashes
+  def diff(with, on = self, level: 0, parents: [])
+    return if with.nil? and on.nil?
+    if with.nil? or on.nil? or with.class != on.class
+      return # XXX ...however we're flagging differences
+    end
+    return if on == with
+
+    tree = ""
+    indentsize = 0
+    parents.each { |p|
+      tree += (" " * indentsize) + p + " => \n"
+      indentsize += 2
+    }
+    indent = (" " * indentsize)
+
+    changes = []
+    if on.is_a?(Hash)
+      on_unique = (on.keys - with.keys)
+      with_unique = (with.keys - on.keys)
+      shared = (with.keys & on.keys)
+      shared.each { |k|
+        diff(with[k], on[k], level: level+1, parents: parents + [k])
+      }
+      on_unique.each { |k|
+        changes << "- ".red+PP.pp({k => on[k] }, '')
+      }
+      with_unique.each { |k|
+        changes << "+ ".green+PP.pp({k => with[k]}, '')
+      }
+    elsif on.is_a?(Array)
+      return if with == on
+      # special case- Basket of Kittens lists of declared resources of a type;
+      # we use this to decide if we can compare two array elements as if they
+      # should be equivalent
+      # We also implement comparison operators for {Hash} and our various
+      # custom objects which we might find in here so that we can get away with
+      # sorting arrays full of weird, non-primitive types.
+      done = []
+#      before_a = on.dup
+#      after_a = on.dup.sort
+#      before_b = with.dup
+#      after_b = with.dup.sort
+      on.sort.each { |elt|
+        if elt.is_a?(Hash) and elt['name'] or elt['entity']# or elt['cloud_id']
+          with.sort.each { |other_elt|
+            if (elt['name'] and other_elt['name'] == elt['name']) or
+               (elt['name'].nil? and !elt["id"].nil? and elt["id"] == other_elt["id"]) or
+               (elt['name'].nil? and elt["id"].nil? and
+                !elt["entity"].nil? and !other_elt["entity"].nil? and
+                 (
+                   (elt["entity"]["id"] and elt["entity"]["id"] == other_elt["entity"]["id"]) or
+                   (elt["entity"]["name"] and elt["entity"]["name"] == other_elt["entity"]["name"])
+                 )
+               )
+              break if elt == other_elt
+              done << elt
+              done << other_elt
+              namestr = if elt['type']
+                "#{elt['type']}[#{elt['name']}]"
+              elsif elt['name']
+                elt['name']
+              elsif elt['entity'] and elt["entity"]["id"]
+                elt['entity']['id']
+              end
+
+              diff(other_elt, elt, level: level+1, parents: parents + [namestr])
+              break
+            end
+          }
+        end
+      }
+      on_unique = (on - with) - done
+      with_unique = (with - on) - done
+#    if on_unique.size > 0 or with_unique.size > 0
+#      if before_a != after_a
+#        MU.log "A BEFORE", MU::NOTICE, details: before_a
+#        MU.log "A AFTER", MU::NOTICE, details: after_a
+#      end
+#      if before_b != after_b
+#        MU.log "B BEFORE", MU::NOTICE, details: before_b
+#        MU.log "B AFTER", MU::NOTICE, details: after_b
+#      end
+#    end
+      on_unique.each { |e|
+        changes << if e.is_a?(Hash)
+          "- ".red+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
+        else
+          "- ".red+e.to_s
+        end
+      }
+      with_unique.each { |e|
+        changes << if e.is_a?(Hash)
+          "+ ".green+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
+        else
+          "+ ".green+e.to_s
+        end
+      }
+    else
+      if on != with
+        changes << "-".red+" #{on.to_s}"
+        changes << "+".green+" #{with.to_s}"
+      end
+    end
+
+    if changes.size > 0
+      puts tree
+      changes.each { |c|
+        puts indent+c
+      }
+    end
+  end
+
+  # Implement a merge! that just updates each hash leaf as needed, not 
+  # trashing the branch on the way there.
+  def deep_merge!(with, on = self)
+
+    if on and with and with.is_a?(Hash)
+      with.each_pair { |k, v|
+        if !on[k] or !on[k].is_a?(Hash)
+          on[k] = v
+        else
+          deep_merge!(with[k], on[k])
+        end
+      }
+    elsif with
+      on = with
+    end
+
+    on
+  end
+end
+
 ENV['HOME'] = Etc.getpwuid(Process.uid).dir
 
 require 'mu/logger'
 module MU
 
+  # Subclass core thread so we can gracefully handle it when we hit system
+  # thread limits. Back off and wait makes sense for us, since most of our
+  # threads are terminal (in the dependency sense) and this is unlikely to get
+  # us deadlocks.
+  class Thread < ::Thread
+    @@mu_global_threads = []
+    @@mu_global_thread_semaphore = Mutex.new
+
+    def initialize(*args, &block)
+      @@mu_global_thread_semaphore.synchronize {
+        @@mu_global_threads.reject! { |t| t.nil? or !t.status }
+      }
+      newguy = nil
+      start = Time.now
+      begin
+        newguy = super(*args, &block)
+        if newguy.nil?
+          MU.log "I somehow got a nil trying to create a thread", MU::WARN, details: caller
+          sleep 1
+        end
+      rescue ::ThreadError => e
+        if e.message.match(/Resource temporarily unavailable/)
+          toomany = @@mu_global_threads.size
+          MU.log "Hit the wall at #{toomany.to_s} threads, waiting until there are fewer", MU::WARN
+          if @@mu_global_threads.size >= toomany
+            sleep 1
+            begin
+              @@mu_global_thread_semaphore.synchronize {
+                @@mu_global_threads.each { |t|
+                  next if t == ::Thread.current
+                  t.join(0.1)
+                }
+                @@mu_global_threads.reject! { |t| t.nil? or !t.status }
+              }
+              if (Time.now - start) > 150
+                MU.log "Failed to get a free thread slot after 150 seconds- are we in a deadlock situation?", MU::ERR, details: caller
+                raise e
+              end
+            end while @@mu_global_threads.size >= toomany
+          end
+          retry
+        else
+          raise e
+        end
+      end while newguy.nil?
+
+      @@mu_global_thread_semaphore.synchronize {
+        @@mu_global_threads << newguy
+      }
+
+    end
+  end
+
   # Wrapper class for fatal Exceptions. Gives our internals something to
   # inherit that will log an error message appropriately before bubbling up.
   class MuError < StandardError
     def initialize(message = nil)
-      MU.log message, MU::ERR if !message.nil?
+      MU.log message, MU::ERR, details: caller[2] if !message.nil?
       if MU.verbosity == MU::Logger::SILENT
-        super
-      else
         super ""
+      else
+        super message
       end
     end
   end
@@ -60,9 +292,9 @@ module MU
     def initialize(message = nil)
       MU.log message, MU::NOTICE if !message.nil?
       if MU.verbosity == MU::Logger::SILENT
-        super
-      else
         super ""
+      else
+        super message
       end
     end
   end
@@ -80,11 +312,37 @@ module MU
     @@myRoot
   end
 
+  # utility routine for sorting semantic versioning strings
+  def self.version_sort(a, b)
+    a_parts = a.split(/[^a-z0-9]/)
+    b_parts = b.split(/[^a-z0-9]/)
+    for i in 0..a_parts.size
+      matchval = if a_parts[i] and b_parts[i] and
+                    a_parts[i].match(/^\d+/) and b_parts[i].match(/^\d+/)
+        a_parts[i].to_i <=> b_parts[i].to_i
+      elsif a_parts[i] and !b_parts[i]
+        1
+      elsif !a_parts[i] and b_parts[i]
+        -1
+      else
+        a_parts[i] <=> b_parts[i]
+      end
+      return matchval if matchval != 0
+    end
+    0
+  end
+
   # Front our global $MU_CFG hash with a read-only copy
   def self.muCfg
     Marshal.load(Marshal.dump($MU_CFG)).freeze
   end
 
+  # Returns true if we're running without a full systemwide Mu Master install,
+  # typically as a gem.
+  def self.localOnly
+    ((Gem.paths and Gem.paths.home and File.realpath(File.expand_path(File.dirname(__FILE__))).match(/^#{Gem.paths.home}/)) or !Dir.exist?("/opt/mu"))
+  end
+
   # The main (root) Mu user's data directory.
   @@mainDataDir = File.expand_path(@@myRoot+"/../var")
   # The main (root) Mu user's data directory.
@@ -133,6 +391,7 @@ module MU
   # Copy the set of global variables in use by another thread, typically our
   # parent thread.
   def self.dupGlobals(parent_thread_id)
+    @@globals[parent_thread_id] ||= {}
     @@globals[parent_thread_id].each_pair { |name, value|
       setVar(name, value)
     }
@@ -215,13 +474,15 @@ module MU
   @myDataDir = @@mainDataDir if @myDataDir.nil?
   # Mu's deployment metadata directory.
   def self.dataDir(for_user = MU.mu_user)
-    if for_user.nil? or for_user.empty? or for_user == "mu" or for_user == "root"
+    if !localOnly and
+       ((Process.uid == 0 and (for_user.nil? or for_user.empty?)) or
+        for_user == "mu" or for_user == "root")
       return @myDataDir
     else
       for_user ||= MU.mu_user
       basepath = Etc.getpwnam(for_user).dir+"/.mu"
-      Dir.mkdir(basepath, 0755) if !Dir.exists?(basepath)
-      Dir.mkdir(basepath+"/var", 0755) if !Dir.exists?(basepath+"/var")
+      Dir.mkdir(basepath, 0755) if !Dir.exist?(basepath)
+      Dir.mkdir(basepath+"/var", 0755) if !Dir.exist?(basepath+"/var")
       return basepath+"/var"
     end
   end
@@ -233,14 +494,24 @@ module MU
     end
     @@globals[Thread.current.object_id]['verbosity']
   end
+  
+  # The color logging flag merits a default value.
+  def self.color
+    if @@globals[Thread.current.object_id].nil? or @@globals[Thread.current.object_id]['color'].nil?
+      MU.setVar("color", true)
+    end
+    @@globals[Thread.current.object_id]['color']
+  end
 
   # Set parameters parameters for calls to {MU#log}
-  def self.setLogging(verbosity, webify_logs = false, handle = STDOUT)
+  def self.setLogging(verbosity, webify_logs = false, handle = STDOUT, color = true)
     MU.setVar("verbosity", verbosity)
-    @@logger ||= MU::Logger.new(verbosity, webify_logs, handle)
+    MU.setVar("color", color)
+    @@logger ||= MU::Logger.new(verbosity, webify_logs, handle, color)
     @@logger.html = webify_logs
     @@logger.verbosity = verbosity
     @@logger.handle = handle
+    @@logger.color = color
   end
 
   setLogging(MU::Logger::NORMAL, false)
@@ -252,7 +523,7 @@ module MU
   end
 
   # Shortcut to invoke {MU::Logger#log}
-  def self.log(msg, level = MU::INFO, details: nil, html: html = false, verbosity: MU.verbosity)
+  def self.log(msg, level = MU::INFO, details: nil, html: false, verbosity: MU.verbosity, color: true)
     return if (level == MU::DEBUG and verbosity <= MU::Logger::LOUD)
     return if verbosity == MU::Logger::SILENT
 
@@ -275,9 +546,9 @@ module MU
         extra = Hash.new if extra.nil?
         extra[:details] = details
       end
-      @@logger.log(msg, level, details: extra, verbosity: MU::Logger::LOUD, html: html)
+      @@logger.log(msg, level, details: extra, verbosity: MU::Logger::LOUD, html: html, color: color)
     else
-      @@logger.log(msg, level, html: html, verbosity: verbosity)
+      @@logger.log(msg, level, html: html, verbosity: verbosity, color: color)
     end
   end
 
@@ -308,49 +579,66 @@ module MU
   require 'mu/groomer'
 
   # Little hack to initialize library-only environments' config files
-  if !$MU_CFG
-    require "#{@@myRoot}/bin/mu-load-config.rb"
-
-    if !$MU_CFG['auto_detection_done'] and (!$MU_CFG['multiuser'] or !cfgExists?)
-      MU.log "Auto-detecting cloud providers"
-      new_cfg = $MU_CFG.dup
-      examples = {}
-      MU::Cloud.supportedClouds.each { |cloud|
-        cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-        begin
-          if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
-            cfg_blob = cloudclass.hosted_config
-            if cfg_blob
-              new_cfg[cloud.downcase] = cfg_blob
-              MU.log "Adding #{cloud} stanza to #{cfgPath}", MU::NOTICE
-            end
-          elsif !$MU_CFG[cloud.downcase] and !cloudclass.config_example.nil?
-            examples[cloud.downcase] = cloudclass.config_example
+  def self.detectCloudProviders
+    MU.log "Auto-detecting cloud providers"
+    new_cfg = $MU_CFG.dup
+    examples = {}
+    MU::Cloud.supportedClouds.each { |cloud|
+      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+      begin
+        if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
+          cfg_blob = cloudclass.hosted_config
+          if cfg_blob
+            new_cfg[cloud.downcase] = cfg_blob
+            MU.log "Adding auto-detected #{cloud} stanza", MU::NOTICE
           end
-        rescue NoMethodError => e
-          # missing .hosted? is normal for dummy layers like CloudFormation
-          MU.log e.message, MU::WARN
+        elsif !$MU_CFG[cloud.downcase] and !cloudclass.config_example.nil?
+          examples[cloud.downcase] = cloudclass.config_example
         end
-      }
-      new_cfg['auto_detection_done'] = true
-      if new_cfg != $MU_CFG or !cfgExists?
-        MU.log "Generating #{cfgPath}"
-        saveMuConfig(new_cfg, examples) # XXX and reload it
+      rescue NoMethodError => e
+        # missing .hosted? is normal for dummy layers like CloudFormation
+        MU.log e.message, MU::WARN
       end
+    }
+    new_cfg['auto_detection_done'] = true
+    if new_cfg != $MU_CFG or !cfgExists?
+      MU.log "Generating #{cfgPath}"
+      saveMuConfig(new_cfg, examples) # XXX and reload it
     end
+    new_cfg
+  end
+
+  if !$MU_CFG
+    require "#{@@myRoot}/bin/mu-load-config.rb"
+    if !$MU_CFG['auto_detection_done'] and (!$MU_CFG['multiuser'] or !cfgExists?)
+      detectCloudProviders
+    end
+  end
+
+  @@mommacat_port = 2260
+  if !$MU_CFG.nil? and !$MU_CFG['mommacat_port'].nil? and
+     !$MU_CFG['mommacat_port'] != "" and $MU_CFG['mommacat_port'].to_i > 0 and
+     $MU_CFG['mommacat_port'].to_i < 65536
+    @@mommacat_port = $MU_CFG['mommacat_port'].to_i
+  end
+  # The port on which the Momma Cat daemon should listen for requests
+  # @return [Integer]
+  def self.mommaCatPort
+    @@mommacat_port
   end
 
   @@my_private_ip = nil
   @@my_public_ip = nil
   @@mu_public_addr = nil
   @@mu_public_ip = nil
-  if $MU_CFG['aws'] # XXX this should be abstracted to elsewhere
+  if MU::Cloud::AWS.hosted?
     @@my_private_ip = MU::Cloud::AWS.getAWSMetaData("local-ipv4")
     @@my_public_ip = MU::Cloud::AWS.getAWSMetaData("public-ipv4")
     @@mu_public_addr = @@my_public_ip
     @@mu_public_ip = @@my_public_ip
   end
-  if !$MU_CFG.nil? and !$MU_CFG['public_address'].nil? and !$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
+  if !$MU_CFG.nil? and !$MU_CFG['public_address'].nil? and
+     !$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
     @@mu_public_addr = $MU_CFG['public_address']
     if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/)
       resolver = Resolv::DNS.new
@@ -400,7 +688,9 @@ module MU
   def self.userEmail(user = MU.mu_user)
     @userlist ||= MU::Master.listUsers
     user = "mu" if user == "root"
-    if Dir.exists?("#{MU.mainDataDir}/users/#{user}")
+    if Dir.exist?("#{MU.mainDataDir}/users/#{user}") and
+       File.readable?("#{MU.mainDataDir}/users/#{user}/email") and
+       File.size?("#{MU.mainDataDir}/users/#{user}/email")
       return File.read("#{MU.mainDataDir}/users/#{user}/email").chomp
     elsif @userlist.has_key?(user)
       return @userlist[user]['email']
@@ -413,7 +703,9 @@ module MU
   # Fetch the real-world name of a given Mu user
   def self.userName(user = MU.mu_user)
     @userlist ||= MU::Master.listUsers
-    if Dir.exists?("#{MU.mainDataDir}/users/#{user}")
+    if Dir.exist?("#{MU.mainDataDir}/users/#{user}") and
+       File.readable?("#{MU.mainDataDir}/users/#{user}/realname") and
+       File.size?("#{MU.mainDataDir}/users/#{user}/realname")
       return File.read("#{MU.mainDataDir}/users/#{user}/realname").chomp
     elsif @userlist.has_key?(user)
       return @userlist[user]['email']
@@ -430,6 +722,14 @@ module MU
     ["Chef", "Ansible"]
   end
 
+  # The version of Chef we will install on nodes.
+  @@chefVersion = "14.0.190"
+  # The version of Chef we will install on nodes.
+  # @return [String]
+  def self.chefVersion
+    @@chefVersion
+  end
+
   MU.supportedGroomers.each { |groomer|
     require "mu/groomers/#{groomer.downcase}"
   }
@@ -451,6 +751,8 @@ module MU
       @@myRegion_var = zone.gsub(/^.*?\/|\-\d+$/, "")
     elsif MU::Cloud::AWS.hosted?
       @@myRegion_var ||= MU::Cloud::AWS.myRegion
+    elsif MU::Cloud::Azure.hosted?
+      @@myRegion_var ||= MU::Cloud::Azure.myRegion
     else
       @@myRegion_var = nil
     end
@@ -458,6 +760,7 @@ module MU
   end
 
   require 'mu/config'
+  require 'mu/adoption'
 
   # Figure out what cloud provider we're in, if any.
   # @return [String]: Google, AWS, etc. Returns nil if we don't seem to be in a cloud.
@@ -468,6 +771,10 @@ module MU
     elsif MU::Cloud::AWS.hosted?
       @@myInstanceId = MU::Cloud::AWS.getAWSMetaData("instance-id")
       return "AWS"
+    elsif MU::Cloud::Azure.hosted?
+      metadata = MU::Cloud::Azure.get_metadata()["compute"]
+      @@myInstanceId = MU::Cloud::Azure::Id.new("/subscriptions/"+metadata["subscriptionId"]+"/resourceGroups/"+metadata["resourceGroupName"]+"/providers/Microsoft.Compute/virtualMachines/"+metadata["name"])
+      return "Azure"
     end
     nil
   end
@@ -513,63 +820,105 @@ module MU
     @@myAZ_var
   end
 
-  @@myCloudDescriptor = nil
-  if MU::Cloud::Google.hosted?
-    @@myCloudDescriptor = MU::Cloud::Google.compute.get_instance(
-      MU::Cloud::Google.myProject,
-      MU.myAZ,
-      MU.myInstanceId
-    )
-  elsif MU::Cloud::AWS.hosted?
+  # Recursively turn a Ruby OpenStruct into a Hash
+  # @param struct [OpenStruct]
+  # @param stringify_keys [Boolean]
+  # @return [Hash]
+  def self.structToHash(struct, stringify_keys: false)
+    google_struct = false
     begin
-      @@myCloudDescriptor = MU::Cloud::AWS.ec2(region: MU.myRegion).describe_instances(instance_ids: [MU.myInstanceId]).reservations.first.instances.first
-    rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
-    rescue Aws::Errors::MissingCredentialsError => e
-      MU.log "I'm hosted in AWS, but I can't make API calls. Does this instance have an appropriate IAM profile?", MU::WARN
+      google_struct = struct.class.ancestors.include?(::Google::Apis::Core::Hashable)
+    rescue NameError
     end
-  end
 
+    aws_struct = false
+    begin
+      aws_struct = struct.class.ancestors.include?(::Seahorse::Client::Response)
+    rescue NameError
+    end
 
-  @@myVPC_var = nil
-  # The VPC/Network in which this Mu master resides
-  # XXX account for Google and non-cloud situations
-  def self.myVPC
-    return nil if MU.myCloudDescriptor.nil?
+    azure_struct = false
     begin
-      if MU::Cloud::AWS.hosted?
-        @@myVPC_var ||= MU.myCloudDescriptor.vpc_id
-      elsif MU::Cloud::Google.hosted?
-        @@myVPC_var = MU.myCloudDescriptor.network_interfaces.first.network.gsub(/.*?\/([^\/]+)$/, '\1')
+      azure_struct = struct.class.ancestors.include?(::MsRestAzure) or struct.class.name.match(/Azure::.*?::Mgmt::.*?::Models::/)
+    rescue NameError
+    end
+
+    if struct.is_a?(Struct) or struct.class.ancestors.include?(Struct) or
+       google_struct or aws_struct or azure_struct
+
+      hash = if azure_struct
+        MU::Cloud::Azure.respToHash(struct)
       else
-        nil
+        struct.to_h
+      end
+
+      if stringify_keys
+        newhash = {}
+        hash.each_pair { |k, v|
+          newhash[k.to_s] = v
+        }
+        hash = newhash 
       end
-    rescue Aws::EC2::Errors::InternalError => e
-      MU.log "Got #{e.inspect} on MU::Cloud::AWS.ec2(region: #{MU.myRegion}).describe_instances(instance_ids: [#{@@myInstanceId}])", MU::WARN
-      sleep 10
+
+      hash.each_pair { |key, value|
+        hash[key] = self.structToHash(value, stringify_keys: stringify_keys)
+      }
+      return hash
+    elsif struct.is_a?(MU::Config::Ref)
+      struct = struct.to_h
+    elsif struct.is_a?(MU::Cloud::Azure::Id)
+      struct = struct.to_s
+    elsif struct.is_a?(Hash)
+      if stringify_keys
+        newhash = {}
+        struct.each_pair { |k, v|
+          newhash[k.to_s] = v
+        }
+        struct = newhash 
+      end
+      struct.each_pair { |key, value|
+        struct[key] = self.structToHash(value, stringify_keys: stringify_keys)
+      }
+      return struct
+    elsif struct.is_a?(Array)
+      struct.map! { |elt|
+        self.structToHash(elt, stringify_keys: stringify_keys)
+      }
+    elsif struct.is_a?(String)
+      # Cleanse weird encoding problems
+      return struct.dup.to_s.force_encoding("ASCII-8BIT").encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+    else
+      return struct
     end
-    @@myVPC_var
   end
 
-  @@mySubnets_var = nil
-  # The AWS Subnets associated with the VPC this MU Master is in
-  # XXX account for Google and non-cloud situations
-  def self.mySubnets
-    @@mySubnets_var ||= MU::Cloud::AWS.ec2(region: MU.myRegion).describe_subnets(
-      filters: [
-        {
-          name: "vpc-id", 
-          values: [MU.myVPC]
-        }
-      ]
-    ).subnets
+  @@myCloudDescriptor = nil
+  if MU.myCloud
+    found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
+    if !found.nil? and found.size == 1
+      @@myCloudDescriptor = found.first.cloud_desc
+    end
   end
 
-  # The version of Chef we will install on nodes.
-  @@chefVersion = "14.0.190"
-  # The version of Chef we will install on nodes.
-  # @return [String]
-  def self.chefVersion;
-    @@chefVersion
+
+  @@myVPCObj_var = nil
+  # The VPC/Network in which this Mu master resides
+  def self.myVPCObj
+    return nil if MU.myCloud.nil?
+    return @@myVPCObj_var if @@myVPCObj_var
+    cloudclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud)
+    @@myVPCObj_var ||= cloudclass.myVPCObj
+    @@myVPCObj_var
+  end
+
+  @@myVPC_var = nil
+  # The VPC/Network in which this Mu master resides
+  def self.myVPC
+    return nil if MU.myCloud.nil?
+    return @@myVPC_var if @@myVPC_var
+    my_vpc_desc = MU.myVPCObj
+    @@myVPC_var ||= my_vpc_desc.cloud_id if my_vpc_desc
+    @@myVPC_var
   end
 
   # Mu's SSL certificate directory
@@ -590,20 +939,20 @@ module MU
   # @return [Boolean]
   def self.hashCmp(hash1, hash2, missing_is_default: false)
     return false if hash1.nil?
-    hash2.each_pair { |k, v|
+    hash2.keys.each { |k|
       if hash1[k].nil?
         return false
       end
     }
     if !missing_is_default
-      hash1.each_pair { |k, v|
+      hash1.keys.each { |k|
         if hash2[k].nil?
           return false
         end
       }
     end
 
-    hash1.each_pair { |k, v|
+    hash1.keys.each { |k|
       if hash1[k].is_a?(Array) 
         return false if !missing_is_default and hash2[k].nil?
         if !hash2[k].nil?
@@ -654,75 +1003,21 @@ module MU
   end
 
 
-  # Recursively turn a Ruby OpenStruct into a Hash
-  # @param struct [OpenStruct]
-  # @param stringify_keys [Boolean]
-  # @return [Hash]
-  def self.structToHash(struct, stringify_keys: false)
-    google_struct = false
-    begin
-      google_struct = struct.class.ancestors.include?(::Google::Apis::Core::Hashable)
-    rescue NameError
-    end
-
-    aws_struct = false
-    begin
-      aws_struct = struct.class.ancestors.include?(::Seahorse::Client::Response)
-    rescue NameError
-    end
-
-    if struct.is_a?(Struct) or struct.class.ancestors.include?(Struct) or
-       google_struct or aws_struct
-
-      hash = struct.to_h
-      if stringify_keys
-        newhash = {}
-        hash.each_pair { |k, v|
-          newhash[k.to_s] = v
-        }
-        hash = newhash 
-      end
-
-      hash.each_pair { |key, value|
-        hash[key] = self.structToHash(value, stringify_keys: stringify_keys)
-      }
-      return hash
-    elsif struct.is_a?(Hash)
-      if stringify_keys
-        newhash = {}
-        struct.each_pair { |k, v|
-          newhash[k.to_s] = v
-        }
-        struct = newhash 
-      end
-      struct.each_pair { |key, value|
-        struct[key] = self.structToHash(value, stringify_keys: stringify_keys)
-      }
-      return struct
-    elsif struct.is_a?(Array)
-      struct.map! { |elt|
-        self.structToHash(elt, stringify_keys: stringify_keys)
-      }
-    else
-      return struct
-    end
-  end
-
   # Generate a random password which will satisfy the complexity requirements of stock Amazon Windows AMIs.
   # return [String]: A password string.
-  def self.generateWindowsPassword
+  def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 25)
     # We have dopey complexity requirements, be stringent here.
     # I'll be nice and not condense this into one elegant-but-unreadable regular expression
     attempts = 0
-    safe_metachars = Regexp.escape('~!@#%^&*_-+=`|(){}[]:;<>,.?')
+    safe_metachars = Regexp.escape(safe_pattern)
     begin
-      if attempts > 25
+      if attempts > retries
         MU.log "Failed to generate an adequate Windows password after #{attempts}", MU::ERR
         raise MuError, "Failed to generate an adequate Windows password after #{attempts}"
       end
       winpass = Password.random(14..16)
       attempts += 1
-    end while winpass.nil? or !winpass.match(/[A-Z]/) or !winpass.match(/[a-z]/) or !winpass.match(/\d/) or !winpass.match(/[#{safe_metachars}]/) or winpass.match(/[^\w\d#{safe_metachars}]/)
+    end while winpass.nil? or !winpass.match(/^[a-z]/i) or !winpass.match(/[A-Z]/) or !winpass.match(/[a-z]/) or !winpass.match(/\d/) or !winpass.match(/[#{safe_metachars}]/) or winpass.match(/[^\w\d#{safe_metachars}]/)
 
     MU.log "Generated Windows password after #{attempts} attempts", MU::DEBUG
     return winpass