cloud-mu 3.1.6 → 3.4.0

data/modules/mu/mommacat/daemon.rb

@@ -33,7 +33,7 @@ module MU
       my_key = OpenSSL::PKey::RSA.new(@private_key)
 
       begin
-        if my_key.private_decrypt(ciphertext).force_encoding("UTF-8") == @deploy_secret.force_encoding("UTF-8")
+        if my_key.private_decrypt(ciphertext).force_encoding("UTF-8").chomp == @deploy_secret.force_encoding("UTF-8").chomp
           MU.log "Matched ciphertext for #{MU.deploy_id}", MU::INFO
           return true
         else
@@ -165,11 +165,11 @@ module MU
         if e.class.name != "MU::Cloud::AWS::Server::BootstrapTempFail" and !File.exist?(deploy_dir+"/.cleanup."+cloud_id) and !File.exist?(deploy_dir+"/.cleanup")
           MU.log "Grooming FAILED for #{kitten.mu_name} (#{e.inspect})", MU::ERR, details: e.backtrace
           sendAdminSlack("Grooming FAILED for `#{kitten.mu_name}` with `#{e.message}` :crying_cat_face:", msg: e.backtrace.join("\n"))
-         sendAdminMail("Grooming FAILED for #{kitten.mu_name} on #{MU.appname} \"#{MU.handle}\" (#{MU.deploy_id})",
-           msg: e.inspect,
-           data: e.backtrace,
-           debug: true
-         )
+          sendAdminMail("Grooming FAILED for #{kitten.mu_name} on #{MU.appname} \"#{MU.handle}\" (#{MU.deploy_id})",
+            msg: e.inspect,
+            data: e.backtrace,
+            debug: true
+          )
           raise e if reraise_fail
         else
           MU.log "Grooming of #{kitten.mu_name} interrupted by cleanup or planned reboot"
@@ -288,8 +288,8 @@ module MU
 
     # Path to the PID file used by the Momma Cat daemon
     # @return [String]
-    def self.daemonPidFile
-      base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
+    def self.daemonPidFile(root = false)
+      base = ((Process.uid == 0 or root) and !MU.localOnly) ? "/var" : MU.dataDir
       "#{base}/run/mommacat.pid"
     end
 
@@ -306,8 +306,14 @@ module MU
           Dir.mkdir(dir)
         end
       }
-      return 0 if status
+      if (Process.uid != 0 and
+           (!$MU_CFG['overridden_keys'] or !$MU_CFG['overridden_keys'].include?("mommacat_port")) and
+            status(true)
+         ) or status
+        return 0
+      end
     
+      File.unlink(daemonPidFile) if File.exists?(daemonPidFile)
       MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}"
       origdir = Dir.getwd
       Dir.chdir(MU.myRoot+"/modules")
@@ -342,22 +348,25 @@ module MU
       return $?.exitstatus
     end
 
+    @@notified_on_pid = {}
+
     # Return true if the Momma Cat daemon appears to be running
     # @return [Boolean]
-    def self.status
+    def self.status(root = false)
       if MU.inGem? and MU.muCfg['disable_mommacat']
         return true
       end
-      if File.exist?(daemonPidFile)
-        pid = File.read(daemonPidFile).chomp.to_i
+      if File.exist?(daemonPidFile(root))
+        pid = File.read(daemonPidFile(root)).chomp.to_i
         begin
           Process.getpgid(pid)
-          MU.log "Momma Cat running with pid #{pid.to_s}"
+          MU.log "Momma Cat running with pid #{pid.to_s}", (@@notified_on_pid[pid] ? MU::DEBUG : MU::INFO) # shush
+          @@notified_on_pid[pid] = true
           return true
         rescue Errno::ESRCH
         end
       end
-      MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile
+      MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile(root)
       false
     end
     

data/modules/mu/mommacat/naming.rb

@@ -19,6 +19,16 @@ module MU
   # the normal synchronous deploy sequence invoked by *mu-deploy*.
   class MommaCat
 
+    # Lookup table to translate the word "habitat" back to its
+    # provider-specific jargon
+    HABITAT_SYNONYMS = {
+      "AWS" => "account",
+      "CloudFormation" => "account",
+      "Google" => "project",
+      "Azure" => "subscription",
+      "VMWare" => "sddc"
+    }
+
     # Given a cloud provider's native descriptor for a resource, make some
     # reasonable guesses about what the thing's name should be.
     def self.guessName(desc, resourceclass, cloud_id: nil, tag_value: nil)
@@ -47,6 +57,74 @@ module MU
 
     end
 
+    # Given a piece of a BoK resource descriptor Hash, come up with shorthand
+    # strings to give it a name for human readers. If nothing reasonable can be
+    # extracted, returns nil.
+    # @param obj [Hash]
+    # @param array_of [String]
+    # @param habitat_translate [String]
+    # @return [Array<String,nil>]
+    def self.getChunkName(obj, array_of = nil, habitat_translate: nil)
+      return [nil, nil] if obj.nil?
+      if [String, Integer, Boolean].include?(obj.class)
+        return [obj, nil]
+      end
+      obj_type = array_of || obj['type']
+      obj_name = obj['name'] || obj['id'] || obj['mu_name'] || obj['cloud_id']
+
+      name_string = if obj_name
+        if obj_type
+          "#{obj_type}[#{obj_name}]"
+        else
+          obj_name.dup
+        end
+      else
+        found_it = nil
+        using = nil
+        ["entity", "role"].each { |subtype|
+          if obj[subtype] and obj[subtype].is_a?(Hash)
+            found_it = if obj[subtype]["id"]
+              obj[subtype]['id'].dup
+            elsif obj[subtype]["type"] and obj[subtype]["name"]
+              "#{obj[subtype]['type']}[#{obj[subtype]['name']}]"
+            end
+            break
+          end
+        }
+        found_it
+      end
+      if name_string
+        name_string.gsub!(/\[.+?\](\[.+?\]$)/, '\1')
+        if habitat_translate and HABITAT_SYNONYMS[habitat_translate]
+          name_string.sub!(/^habitats?\[(.+?)\]/i, HABITAT_SYNONYMS[habitat_translate]+'[\1]')
+        end
+      end
+
+      location_list = []
+
+      location = if obj['project']
+        obj['project']
+      elsif obj['habitat'] and (obj['habitat']['id'] or obj['habitat']['name'])
+        obj['habitat']['name'] || obj['habitat']['id']
+      else
+        hab_str = nil
+        ['projects', 'habitats'].each { |key|
+
+          if obj[key] and obj[key].is_a?(Array)
+            location_list = obj[key].sort.map { |p|
+              (p["name"] || p["id"]).gsub(/^.*?[^\/]+\/([^\/]+)$/, '\1')
+            }
+            hab_str = location_list.join(", ")
+            name_string.gsub!(/^.*?[^\/]+\/([^\/]+)$/, '\1') if name_string
+            break
+          end
+        }
+        hab_str
+      end
+
+      [name_string, location, location_list]
+    end
+
     # Generate a three-character string which can be used to unique-ify the
     # names of resources which might potentially collide, e.g. Windows local
     # hostnames, Amazon Elastic Load Balancers, or server pool instances.
@@ -218,17 +296,18 @@ module MU
     # SSH config entries, etc.
     # @param server [MU::Cloud::Server]: The {MU::Cloud::Server} we'll be setting up.
     # @param sync_wait [Boolean]: Whether to wait for DNS to fully synchronize before returning.
-    def self.nameKitten(server, sync_wait: false)
+    def self.nameKitten(server, sync_wait: false, no_dns: false)
       node, config, _deploydata = server.describe
 
       mu_zone = nil
       # XXX GCP!
-      if MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
+      if !no_dns and MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
         zones = MU::Cloud::DNSZone.find(cloud_id: "platform-mu")
         mu_zone = zones.values.first if !zones.nil?
       end
+
       if !mu_zone.nil?
-        MU::Cloud::DNSZone.genericMuDNSEntry(name: node, target: server.canonicalIP, cloudclass: MU::Cloud::Server, sync_wait: sync_wait)
+        MU::Cloud::DNSZone.genericMuDNSEntry(name: node.gsub(/[^a-z0-9!"\#$%&'\(\)\*\+,\-\/:;<=>\?@\[\]\^_`{\|}~\.]/, '-').gsub(/--|^-/, ''), target: server.canonicalIP, cloudclass: MU::Cloud::Server, sync_wait: sync_wait)
       else
         MU::Master.addInstanceToEtcHosts(server.canonicalIP, node)
       end

data/modules/mu/mommacat/search.rb

@@ -60,7 +60,7 @@ module MU
       ) 
       _shortclass, _cfg_name, type, _classname, _attrs = MU::Cloud.getResourceNames(type, true)
 
-      cloudclass = MU::Cloud.assertSupportedCloud(cloud)
+      cloudclass = MU::Cloud.cloudClass(cloud)
       return nil if cloudclass.virtual?
 
       if (tag_key and !tag_value) or (!tag_key and tag_value)
@@ -107,7 +107,17 @@ module MU
       matches = []
 
       credlist.each { |creds|
-        cloud_descs = search_cloud_provider(type, cloud, habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
+        cur_habitats = []
+
+        if habitats and !habitats.empty? and habitats != [nil]
+          valid_habitats = cloudclass.listHabitats(creds)
+          cur_habitats = (habitats & valid_habitats)
+          next if cur_habitats.empty?
+        else
+          cur_habitats = cloudclass.listHabitats(creds)
+        end
+
+        cloud_descs = search_cloud_provider(type, cloud, cur_habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
 
         cloud_descs.each_pair.each { |p, regions|
           regions.each_pair.each { |r, results|
@@ -127,6 +137,8 @@ module MU
       matches
     end
 
+    @object_load_fails = false
+
     # Return the resource object of another member of this deployment
     # @param type [String,Symbol]: The type of resource
     # @param name [String]: The name of the resource as defined in its 'name' Basket of Kittens field
@@ -135,7 +147,7 @@ module MU
     # @param created_only [Boolean]: Only return the littermate if its cloud_id method returns a value
     # @param return_all [Boolean]: Return a Hash of matching objects indexed by their mu_name, instead of a single match. Only valid for resource types where has_multiples is true.
     # @return [MU::Cloud]
-    def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, **flags)
+    def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, ignore_missing: false, debug: false, **flags)
       _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type)
 
       # If we specified a habitat, which we may also have done by its shorthand
@@ -159,17 +171,35 @@ module MU
       }
 
       @kitten_semaphore.synchronize {
-        return nil if !@kittens.has_key?(type)
-        matches = []
 
+        if !@kittens.has_key?(type)
+          return nil if !@original_config or @original_config[type].nil? or @original_config[type].empty?
+          begin
+            loadObjects(false)
+          rescue ThreadError => e
+            if e.message !~ /deadlock/
+              raise e
+            end
+          end
+          if @object_load_fails or !@kittens[type]
+            if !ignore_missing
+              MU.log "#{@deploy_id}'s original config has #{@original_config[type].size == 1 ? "a" : @original_config[type].size.to_s} #{type}, but loadObjects could not populate anything from deployment metadata", MU::ERR if !@object_load_fails
+              @object_load_fails = true
+            end
+            return nil
+          end
+        end
+        matches = {}
         @kittens[type].each { |habitat_group, sib_classes|
           next if habitat and habitat_group and habitat_group != habitat
           sib_classes.each_pair { |sib_class, cloud_objs|
+
             if attrs[:has_multiples]
               next if !name.nil? and name != sib_class or cloud_objs.empty?
               if !name.nil?
                 if return_all
-                  return cloud_objs.dup
+                  matches.merge!(cloud_objs.clone)
+                  next
                 elsif cloud_objs.size == 1 and does_match.call(cloud_objs.values.first)
                   return cloud_objs.values.first
                 end
@@ -177,19 +207,24 @@ module MU
               
               cloud_objs.each_value { |obj|
                 if does_match.call(obj)
-                  return (return_all ? cloud_objs.clone : obj.clone)
+                  if return_all
+                    matches.merge!(cloud_objs.clone)
+                  else
+                    return obj.clone
+                  end
                 end
               }
-            # has_multiples is false
+            # has_multiples is false, "cloud_objs" is actually a singular object
             elsif (name.nil? and does_match.call(cloud_objs)) or [sib_class, cloud_objs.virtual_name(name)].include?(name.to_s)
-              matches << cloud_objs.clone
+              matches[cloud_objs.config['name']] = cloud_objs.clone
             end
           }
         }
 
-        return matches.first if matches.size == 1
+        return matches if return_all and matches.size >= 1
+
+        return matches.values.first if matches.size == 1
 
-        return matches if return_all and matches.size > 1
       }
 
       return nil
@@ -213,7 +248,7 @@ module MU
     end
 
     def self.generate_dummy_object(type, cloud, name, mu_name, cloud_id, desc, region, habitat, tag_value, calling_deploy, credentials)
-      resourceclass = MU::Cloud.loadCloudType(cloud, type)
+      resourceclass = MU::Cloud.resourceClass(cloud, type)
 
       use_name = if (name.nil? or name.empty?)
         if !mu_name.nil?
@@ -269,15 +304,23 @@ module MU
     private_class_method :generate_dummy_object
 
     def self.search_cloud_provider(type, cloud, habitats, region, cloud_id: nil, tag_key: nil, tag_value: nil, credentials: nil, flags: nil)
-      cloudclass = MU::Cloud.assertSupportedCloud(cloud)
-      resourceclass = MU::Cloud.loadCloudType(cloud, type)
+      cloudclass = MU::Cloud.cloudClass(cloud)
+      resourceclass = MU::Cloud.resourceClass(cloud, type)
 
       # Decide what regions we'll search, if applicable for this resource
       # type.
       regions = if resourceclass.isGlobal?
         [nil]
       else
-        region ? [region] : cloudclass.listRegions(credentials: credentials)
+        if region
+          if region.is_a?(Array) and !region.empty?
+            region
+          else
+            [region]
+          end
+        else
+          cloudclass.listRegions(credentials: credentials)
+        end
       end
 
       # Decide what habitats (accounts/projects/subscriptions) we'll
@@ -288,7 +331,7 @@ module MU
           habitats << nil
         end
         if resourceclass.canLiveIn.include?(:Habitat)
-          habitats.concat(cloudclass.listHabitats(credentials))
+          habitats.concat(cloudclass.listHabitats(credentials, use_cache: false))
         end
       end
       habitats << nil if habitats.empty?

data/modules/mu/mommacat/storage.rb

@@ -88,16 +88,28 @@ module MU
       }
     end
 
+
     # Overwrite this deployment's configuration with a new version. Save the
     # previous version as well.
     # @param new_conf [Hash]: A new configuration, fully resolved by {MU::Config}
-    def updateBasketofKittens(new_conf)
+    def updateBasketofKittens(new_conf, skip_validation: false, new_metadata: nil, save_now: false)
       loadDeploy
       if new_conf == @original_config
-        MU.log "#{@deploy_id}", MU::WARN
         return
       end
 
+      scrub_with = nil
+
+      # Make sure the new config that we were just handed resolves and makes
+      # sense
+      if !skip_validation
+        f = Tempfile.new(@deploy_id)
+        f.write JSON.parse(JSON.generate(new_conf)).to_yaml
+        conf_engine = MU::Config.new(f.path) # will throw an exception if it's bad, adoption should catch this and cope reasonably
+        scrub_with = conf_engine.config
+        f.close
+      end
+
       backup = "#{deploy_dir}/basket_of_kittens.json.#{Time.now.to_i.to_s}"
       MU.log "Saving previous config of #{@deploy_id} to #{backup}"
       config = File.new(backup, File::CREAT|File::TRUNC|File::RDWR, 0600)
@@ -106,9 +118,27 @@ module MU
       config.flock(File::LOCK_UN)
       config.close
 
-      @original_config = new_conf
-#      save! # XXX this will happen later, more sensibly
-      MU.log "New config saved to #{deploy_dir}/basket_of_kittens.json"
+      @original_config = new_conf.clone
+
+      MU::Cloud.resource_types.each_pair { |res_type, attrs|
+        next if !@deployment.has_key?(attrs[:cfg_plural])
+        deletia = []
+# existing_deploys
+        @deployment[attrs[:cfg_plural]].each_pair { |res_name, data|
+          orig_cfg = findResourceConfig(attrs[:cfg_plural], res_name, (scrub_with || @original_config))
+
+          if orig_cfg.nil? and (!data['mu_name'] or data['mu_name'] =~ /^#{Regexp.quote(@deploy_id)}/)
+            MU.log "#{res_type} #{res_name} no longer configured, will remove deployment metadata", MU::NOTICE, details: data
+            deletia << res_name
+          end
+        }
+        @deployment[attrs[:cfg_plural]].reject! { |k, v| deletia.include?(k) }
+      }
+
+      if save_now
+        save!
+        MU.log "New config saved to #{deploy_dir}/basket_of_kittens.json"
+      end
     end
 
     @lock_semaphore = Mutex.new
@@ -147,11 +177,11 @@ module MU
     # @param id [String]: The lock identifier to release.
     # @param nonblock [Boolean]: Whether to block while waiting for the lock. In non-blocking mode, we simply return false if the lock is not available.
     # return [false, nil]
-    def self.lock(id, nonblock = false, global = false)
+    def self.lock(id, nonblock = false, global = false, retries: 0, deploy_id: MU.deploy_id)
       raise MuError, "Can't pass a nil id to MU::MommaCat.lock" if id.nil?
 
       if !global
-        lockdir = "#{deploy_dir(MU.deploy_id)}/locks"
+        lockdir = "#{deploy_dir(deploy_id)}/locks"
       else
         lockdir = File.expand_path(MU.dataDir+"/locks")
       end
@@ -160,6 +190,7 @@ module MU
         MU.log "Creating #{lockdir}", MU::DEBUG
         Dir.mkdir(lockdir, 0700)
       end
+      nonblock = true if retries > 0
 
       @lock_semaphore.synchronize {
         if @locks[Thread.current.object_id].nil?
@@ -168,11 +199,38 @@ module MU
 
         @locks[Thread.current.object_id][id] = File.open("#{lockdir}/#{id}.lock", File::CREAT|File::RDWR, 0600)
       }
-      MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG
+
+      MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG, details: caller
+      show_relevant = Proc.new {
+        @lock_semaphore.synchronize {
+          @locks.each_pair { |thread_id, lock|
+            lock.each_pair { |lockid, lockpath|
+              if lockid == id
+                thread = Thread.list.select { |t| t.object_id == thread_id }.first
+                if thread.object_id != Thread.current.object_id
+                  MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
+                end
+              end
+            }
+          }
+        }
+      }
       begin
         if nonblock
           if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
-            return false
+            if retries > 0
+              success = false
+              MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) { |cur_retries, _wait|
+                success = @locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
+                if !success and cur_retries > 0 and (cur_retries % 3) == 0
+                  show_relevant.call(cur_retries)
+                end
+              }
+              show_relevant.call(cur_retries) if !success
+              return success
+            else
+              return false
+            end
           end
         else
           @locks[Thread.current.object_id][id].flock(File::LOCK_EX)
@@ -186,11 +244,11 @@ module MU
 
     # Release a flock() lock.
     # @param id [String]: The lock identifier to release.
-    def self.unlock(id, global = false)
+    def self.unlock(id, global = false, deploy_id: MU.deploy_id)
       raise MuError, "Can't pass a nil id to MU::MommaCat.unlock" if id.nil?
       lockdir = nil
       if !global
-        lockdir = "#{deploy_dir(MU.deploy_id)}/locks"
+        lockdir = "#{deploy_dir(deploy_id)}/locks"
       else
         lockdir = File.expand_path(MU.dataDir+"/locks")
       end
@@ -361,6 +419,7 @@ module MU
           deploy.flock(File::LOCK_UN)
           deploy.close
           @need_deploy_flush = false
+          @last_modified = nil
           MU::MommaCat.updateLitter(@deploy_id, self)
         end
 
@@ -512,48 +571,30 @@ module MU
       if !@original_config['scrub_mu_isms'] and !@no_artifacts
         credsets.each_pair { |cloud, creds|
           creds.uniq!
-          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
           creds.each { |credentials|
-            cloudclass.writeDeploySecret(@deploy_id, @deploy_secret, credentials: credentials)
+            MU::Cloud.cloudClass(cloud).writeDeploySecret(@deploy_id, @deploy_secret, credentials: credentials)
           }
         }
       end
     end
 
     def loadObjects(delay_descriptor_load)
+      # Load up MU::Cloud objects for all our kittens in this deploy
+
       MU::Cloud.resource_types.each_pair { |res_type, attrs|
         type = attrs[:cfg_plural]
         next if !@deployment.has_key?(type)
 
+        deletia = {}
         @deployment[type].each_pair { |res_name, data|
-          orig_cfg = nil
-          if @original_config.has_key?(type)
-            @original_config[type].each { |resource|
-              if resource["name"] == res_name
-                orig_cfg = resource
-                break
-              end
-            }
-          end
-
-          # Some Server objects originated from ServerPools, get their
-          # configs from there
-          if type == "servers" and orig_cfg.nil? and
-              @original_config.has_key?("server_pools")
-            @original_config["server_pools"].each { |resource|
-              if resource["name"] == res_name
-                orig_cfg = resource
-                break
-              end
-            }
-          end
+          orig_cfg = findResourceConfig(type, res_name)
 
           if orig_cfg.nil?
             MU.log "Failed to locate original config for #{attrs[:cfg_name]} #{res_name} in #{@deploy_id}", MU::WARN if !["firewall_rules", "databases", "storage_pools", "cache_clusters", "alarms"].include?(type) # XXX shaddap
             next
           end
 
-          if orig_cfg['vpc'] and orig_cfg['vpc'].is_a?(Hash)
+          if orig_cfg['vpc']
             ref = if orig_cfg['vpc']['id'] and orig_cfg['vpc']['id'].is_a?(Hash)
               orig_cfg['vpc']['id']['mommacat'] = self
               MU::Config::Ref.get(orig_cfg['vpc']['id'])
@@ -566,18 +607,12 @@ module MU
           end
 
           begin
-            # Load up MU::Cloud objects for all our kittens in this deploy
-            orig_cfg['environment'] = @environment # not always set in old deploys
             if attrs[:has_multiples]
               data.keys.each { |mu_name|
-                attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: mu_name, delay_descriptor_load: delay_descriptor_load)
+                addKitten(type, res_name, attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: mu_name, delay_descriptor_load: delay_descriptor_load))
               }
             else
-              # XXX hack for old deployments, this can go away some day
-              if data['mu_name'].nil?
-                raise MuError, "Unable to find or guess a Mu name for #{res_type}: #{res_name} in #{@deploy_id}"
-              end
-              attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: data['mu_name'], cloud_id: data['cloud_id'])
+              addKitten(type, res_name, attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: data['mu_name'], cloud_id: data['cloud_id']))
             end
           rescue StandardError => e
             if e.class != MU::Cloud::MuCloudResourceNotImplemented
@@ -585,6 +620,7 @@ module MU
             end
           end
         }
+
       }
     end
 
@@ -612,6 +648,14 @@ module MU
     def loadDeployFromCache(set_context_to_me = true)
       return false if !File.size?(deploy_dir+"/deployment.json")
 
+      lastmod = File.mtime("#{deploy_dir}/deployment.json")
+      if @last_modified and lastmod < @last_modified
+        MU.log "#{deploy_dir}/deployment.json last written at #{lastmod}, live meta at #{@last_modified}, not loading", MU::WARN if @last_modified
+        # this is a weird place for this
+        setThreadContextToMe if set_context_to_me
+        return true
+      end
+
       deploy = File.open("#{deploy_dir}/deployment.json", File::RDONLY)
       MU.log "Getting lock to read #{deploy_dir}/deployment.json", MU::DEBUG
       # deploy.flock(File::LOCK_EX)
@@ -623,6 +667,7 @@ module MU
 
       begin
         @deployment = JSON.parse(File.read("#{deploy_dir}/deployment.json"))
+# XXX is it worthwhile to merge fuckery?
       rescue JSON::ParserError => e
         MU.log "JSON parse failed on #{deploy_dir}/deployment.json", MU::ERR, details: e.message
       end
@@ -632,20 +677,21 @@ module MU
 
       setThreadContextToMe if set_context_to_me
 
-      @timestamp = @deployment['timestamp']
-      @seed = @deployment['seed']
-      @appname = @deployment['appname']
-      @handle = @deployment['handle']
-
       true
     end
 
+
     ###########################################################################
     ###########################################################################
     def loadDeploy(deployment_json_only = false, set_context_to_me: true)
       MU::MommaCat.deploy_struct_semaphore.synchronize {
         success = loadDeployFromCache(set_context_to_me)
 
+        @timestamp ||= @deployment['timestamp']
+        @seed ||= @deployment['seed']
+        @appname ||= @deployment['appname']
+        @handle ||= @deployment['handle']
+
         return if deployment_json_only and success
 
         if File.exist?(deploy_dir+"/private_key")
@@ -687,5 +733,30 @@ module MU
       }
     end
 
+    def findResourceConfig(type, name, config = @original_config)
+      orig_cfg = nil
+      if config.has_key?(type)
+        config[type].each { |resource|
+          if resource["name"] == name
+            orig_cfg = resource
+            break
+          end
+        }
+      end
+  
+      # Some Server objects originated from ServerPools, get their
+      # configs from there
+      if type == "servers" and orig_cfg.nil? and config.has_key?("server_pools")
+        config["server_pools"].each { |resource|
+          if resource["name"] == name
+            orig_cfg = resource
+            break
+          end
+        }
+      end
+
+      orig_cfg
+    end
+
   end #class
 end #module