cloud-mu 3.1.6 → 3.4.0

data/extras/clean-stock-amis

@@ -23,7 +23,7 @@ $opts = Optimist::options do
 #{$0} [-c credentials] [-i imagename]
   EOS
   opt :credentials, "Use these AWS credentials from mu.yaml instead of the default set", :required => false, :type => :string
-  opt :image, "Purge a specific image, instead of just scrubing old ones", :required => false, :type => :string
+  opt :image, "Purge a specific image, instead of just scrubbing old ones", :required => false, :type => :string
 end
 
 filters = [
@@ -33,12 +33,14 @@ filters = [
   }
 ]
 
+in_use = MU::Cloud.getStockImage("AWS").values.flatten.map { |h| h.values }.flatten
 
 MU::Cloud::AWS.listRegions.each { | r|
   images = MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).describe_images(
     filters: filters + [{ "name" => "state", "values" => ["available"]}]
   ).images
   images.each { |ami|
+    next if in_use.include?(ami)
     if ($opts[:image] and ami.name == $opts[:image]) or 
        ((DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/))
       snaps = []
@@ -53,7 +55,13 @@ MU::Cloud::AWS.listRegions.each { | r|
       rescue Aws::EC2::Errors::InvalidAMIIDUnavailable
       end
         snaps.each { |snap_id|
-          MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
+          begin
+            MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
+          rescue Aws::EC2::Errors::InvalidSnapshotInUse
+            sleep 5
+            retry
+          rescue Aws::EC2::Errors::InvalidSnapshotNotFound
+          end
         }
     end
   }

data/extras/generate-stock-images

@@ -63,6 +63,7 @@ end
 now = DateTime.now
 
 exitcode = 0
+succeeded = 0
 $opts[:clouds].each { |cloud|
   current_images = MU::Cloud.getStockImage(cloud, fail_hard: true)
   $opts[:platforms].each { |platform|
@@ -91,6 +92,7 @@ $opts[:clouds].each { |cloud|
         end
         next if !needed
       end
+      MU.log "Loading "+bok_dir+"/"+cloud+"/"+platform+".yaml"
       conf_engine = MU::Config.new(
         bok_dir+"/"+cloud+"/"+platform+".yaml",
         default_credentials: $opts[(cloud.downcase+"_creds").to_sym]
@@ -113,6 +115,7 @@ $opts[:clouds].each { |cloud|
           # Scrub any loose metadata left over from our image deployment. It's
           # ok, this won't touch the images we just made.
           MU::Cleanup.run(deployer.mommacat.deploy_id, skipsnapshots: true, verbosity: MU::Logger::QUIET)
+          succeeded += 1
         rescue Exception => e
           MU.log e.message, MU::ERR
           exitcode = 1
@@ -121,10 +124,11 @@ $opts[:clouds].each { |cloud|
     end
   }
 
-  if !available_clouds.keys.include?("AWS") # XXX or if we don't have permissions
+  if !$opts[:dryrun] and succeeded > 0
     puts current_images.to_yaml
-  elsif !$opts[:dryrun]
-    MU::Cloud::AWS::Bucket.upload($opts[:upload_to]+"/"+cloud+".yaml", data: current_images.to_yaml, credentials: $opts[:aws_creds], acl: "public-read")
+    if available_clouds.keys.include?("AWS")
+      MU::Cloud::AWS::Bucket.upload($opts[:upload_to]+"/"+cloud+".yaml", data: current_images.to_yaml, credentials: $opts[:aws_creds], acl: "public-read")
+    end
   end
 }
 

data/extras/image-generators/AWS/centos7.yaml

@@ -1,17 +1,20 @@
 ---
-  appname: mu
-  servers: 
-    - 
-      name: centos7
-      platform: centos7
-      size: m3.medium
-      scrub_groomer: true
-      run_list:
-      - recipe[mu-tools::apply_security]
-      - recipe[mu-tools::updates]
-      - recipe[mu-tools::split_var_partitions]
-      create_image:
-        image_then_destroy: true
-        public: true
-        copy_to_regions:
-        - "#ALL"
+appname: mu
+servers: 
+- name: centos7
+  platform: centos7
+  size: m4.large
+  vpc:
+    name: c7vpc
+  scrub_groomer: true
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]
+  create_image:
+    image_then_destroy: true
+    public: true
+    copy_to_regions:
+    - "#ALL"
+vpcs:
+- name: c7vpc

data/modules/mommacat.ru

@@ -387,7 +387,7 @@ app = proc do |env|
 
 # XXX We can't assume AWS anymore. What does this look like otherwise?
 # If this is an already-groomed instance, try to get a real object for it
-      instance = MU::MommaCat.findStray("AWS", "server", cloud_id: req["mu_instance_id"], region: server_cfg["region"], deploy_id: req["mu_id"], name: req["mu_resource_name"], dummy_ok: false, calling_deploy: kittenpile).first
+      instance = MU::MommaCat.findStray("AWS", "server", cloud_id: req["mu_instance_id"], region: server_cfg["region"], deploy_id: req["mu_id"], name: req["mu_resource_name"], dummy_ok: true, calling_deploy: kittenpile).first
       mu_name = nil
       if instance.nil?
         # Now we're just checking for existence in the cloud provider, really
@@ -416,7 +416,7 @@ app = proc do |env|
         if instance.respond_to?(:addVolume)
 # XXX make sure we handle mangled input safely
           params = JSON.parse(Base64.decode64(req["add_volume"]))
-MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
+          MU.log "add_volume request", MU::NOTICE, details: params
           instance.addVolume(params["dev"], params["size"], delete_on_termination: params["delete_on_termination"])
         else
           returnval = throw500 "I don't know how to add a volume for #{instance}"

data/modules/mu.rb

@@ -79,38 +79,40 @@ class Hash
     }
     return 0 if self == other # that was easy!
     # compare elements and decide who's "bigger" based on their totals?
-    0
+
+    # fine, try some brute force and just hope everything implements to_s
+    self.flatten.map { |e| e.to_s }.join() <=> other.flatten.map { |e| e.to_s }.join()
   end
 
-  # Recursively compare two hashes
-  def diff(with, on = self, level: 0, parents: [])
+  # Recursively compare two Mu Basket of Kittens hashes and report the differences
+  def diff(with, on = self, level: 0, parents: [], report: {}, habitat: nil)
     return if with.nil? and on.nil?
     if with.nil? or on.nil? or with.class != on.class
       return # XXX ...however we're flagging differences
     end
     return if on == with
 
-    tree = ""
-    indentsize = 0
-    parents.each { |p|
-      tree += (" " * indentsize) + p + " => \n"
-      indentsize += 2
-    }
-    indent = (" " * indentsize)
-
     changes = []
+    report ||= {}
     if on.is_a?(Hash)
       on_unique = (on.keys - with.keys)
       with_unique = (with.keys - on.keys)
       shared = (with.keys & on.keys)
       shared.each { |k|
-        diff(with[k], on[k], level: level+1, parents: parents + [k])
+
+        report_data = diff(with[k], on[k], level: level+1, parents: parents + [k], report: report[k], habitat: habitat)
+        if report_data and !report_data.empty?
+          report ||= {}
+          report[k] = report_data
+        end
       }
       on_unique.each { |k|
-        changes << "- ".red+PP.pp({k => on[k] }, '')
+        report[k] = { :action => :removed, :parents => parents, :value => on[k].clone }
+        report[k][:habitat] = habitat if habitat
       }
       with_unique.each { |k|
-        changes << "+ ".green+PP.pp({k => with[k]}, '')
+        report[k] = { :action => :added, :parents => parents, :value => with[k].clone }
+        report[k][:habitat] = habitat if habitat
       }
     elsif on.is_a?(Array)
       return if with == on
@@ -122,29 +124,27 @@ class Hash
       # sorting arrays full of weird, non-primitive types.
       done = []
       on.sort.each { |elt|
-        if elt.is_a?(Hash) and elt['name'] or elt['entity']# or elt['cloud_id']
-          with.sort.each { |other_elt|
-            # Figure out what convention this thing is using for resource identification
-            compare_a, compare_b = if elt['name'].nil? and elt["id"].nil? and !elt["entity"].nil? and !other_elt["entity"].nil?
-              [elt["entity"], other_elt["entity"]]
-            else
-              [elt, other_elt]
-            end
+        if elt.is_a?(Hash) and !MU::MommaCat.getChunkName(elt).first.nil?
+          elt_namestr, elt_location, elt_location_list = MU::MommaCat.getChunkName(elt)
 
-            if (compare_a['name'] and compare_b['name'] == compare_a['name']) or
-               (compare_a['name'].nil? and !compare_a["id"].nil? and compare_a["id"] == compare_b["id"])
-              break if elt == other_elt
+          with.sort.each { |other_elt|
+            other_elt_namestr, other_elt_location, other_elt_location_list = MU::MommaCat.getChunkName(other_elt)
+
+            # Case 1: The array element exists in both version of this array
+            if elt_namestr and other_elt_namestr and
+               elt_namestr == other_elt_namestr and
+               (elt_location.nil? or other_elt_location.nil? or
+                elt_location == other_elt_location or
+                !(elt_location_list & other_elt_location_list).empty?
+               )
               done << elt
               done << other_elt
-              namestr = if elt['type']
-                "#{elt['type']}[#{elt['name']}]"
-              elsif elt['name']
-                elt['name']
-              elsif elt['entity'] and elt["entity"]["id"]
-                elt['entity']['id']
+              break if elt == other_elt # if they're identical, we're done
+              report_data = diff(other_elt, elt, level: level+1, parents: parents + [elt_namestr], habitat: (elt_location || habitat))
+              if report_data and !report_data.empty?
+                report ||= {}
+                report[elt_namestr] = report_data
               end
-
-              diff(other_elt, elt, level: level+1, parents: parents + [namestr])
               break
             end
           }
@@ -152,43 +152,34 @@ class Hash
       }
       on_unique = (on - with) - done
       with_unique = (with - on) - done
-#    if on_unique.size > 0 or with_unique.size > 0
-#      if before_a != after_a
-#        MU.log "A BEFORE", MU::NOTICE, details: before_a
-#        MU.log "A AFTER", MU::NOTICE, details: after_a
-#      end
-#      if before_b != after_b
-#        MU.log "B BEFORE", MU::NOTICE, details: before_b
-#        MU.log "B AFTER", MU::NOTICE, details: after_b
-#      end
-#    end
+
+      # Case 2: This array entry exists in the old version, but not the new one
       on_unique.each { |e|
-        changes << if e.is_a?(Hash)
-          "- ".red+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
-        else
-          "- ".red+e.to_s
-        end
+        namestr, loc = MU::MommaCat.getChunkName(e)
+
+        report ||= {}
+        report[namestr] = { :action => :removed, :parents => parents, :value => e.clone }
+        report[namestr][:habitat] = loc if loc
       }
+
+      # Case 3: This array entry exists in the new version, but not the old one
       with_unique.each { |e|
-        changes << if e.is_a?(Hash)
-          "+ ".green+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
-        else
-          "+ ".green+e.to_s
-        end
+        namestr, loc = MU::MommaCat.getChunkName(e)
+
+        report ||= {}
+        report[namestr] = { :action => :added, :parents => parents, :value => e.clone }
+        report[namestr][:habitat] = loc if loc
       }
+
+    # A plain old leaf node of data
     else
       if on != with
-        changes << "-".red+" #{on.to_s}"
-        changes << "+".green+" #{with.to_s}"
+        report = { :action => :changed, :parents => parents, :oldvalue => on, :value => with.clone }
+        report[:habitat] = habitat if habitat
       end
     end
 
-    if changes.size > 0
-      puts tree
-      changes.each { |c|
-        puts indent+c
-      }
-    end
+    report.freeze
   end
 
   # Implement a merge! that just updates each hash leaf as needed, not 
@@ -212,8 +203,29 @@ class Hash
 end
 
 ENV['HOME'] = Etc.getpwuid(Process.uid).dir
+module MU
+
+  # For log entries that should only be logged when we're in verbose mode
+  DEBUG = 0.freeze
+  # For ordinary log entries
+  INFO = 1.freeze
+  # For more interesting log entries which are not errors
+  NOTICE = 2.freeze
+  # Log entries for non-fatal errors
+  WARN = 3.freeze
+  # Log entries for non-fatal errors
+  WARNING = 3.freeze
+  # Log entries for fatal errors
+  ERR = 4.freeze
+  # Log entries for fatal errors
+  ERROR = 4.freeze
+  # Log entries that will be held and displayed/emailed at the end of deploy,
+  # cleanup, etc.
+  SUMMARY = 5.freeze
+end
 
 require 'mu/logger'
+
 module MU
 
   # Subclass core thread so we can gracefully handle it when we hit system
@@ -273,8 +285,9 @@ module MU
   # Wrapper class for fatal Exceptions. Gives our internals something to
   # inherit that will log an error message appropriately before bubbling up.
   class MuError < StandardError
-    def initialize(message = nil, silent: false)
-      MU.log message, MU::ERR, details: caller[2] if !message.nil? and !silent
+    def initialize(message = nil, silent: false, details: nil)
+      details ||= caller[2]
+      MU.log message, MU::ERR, details: details if !message.nil? and !silent
       if MU.verbosity == MU::Logger::SILENT
         super ""
       else
@@ -286,8 +299,8 @@ module MU
   # Wrapper class for temporary Exceptions. Gives our internals something to
   # inherit that will log a notice message appropriately before bubbling up.
   class MuNonFatal < StandardError
-    def initialize(message = nil, silent: false)
-      MU.log message, MU::NOTICE if !message.nil? and !silent
+    def initialize(message = nil, silent: false, details: nil)
+      MU.log message, MU::NOTICE, details: details if !message.nil? and !silent
       if MU.verbosity == MU::Logger::SILENT
         super ""
       else
@@ -620,25 +633,6 @@ module MU
     @@logger.log(msg, level, details: details, html: html, verbosity: verbosity, color: color)
   end
 
-  # For log entries that should only be logged when we're in verbose mode
-  DEBUG = 0.freeze
-  # For ordinary log entries
-  INFO = 1.freeze
-  # For more interesting log entries which are not errors
-  NOTICE = 2.freeze
-  # Log entries for non-fatal errors
-  WARN = 3.freeze
-  # Log entries for non-fatal errors
-  WARNING = 3.freeze
-  # Log entries for fatal errors
-  ERR = 4.freeze
-  # Log entries for fatal errors
-  ERROR = 4.freeze
-  # Log entries that will be held and displayed/emailed at the end of deploy,
-  # cleanup, etc.
-  SUMMARY = 5.freeze
-
-
   autoload :Cleanup, 'mu/cleanup'
   autoload :Deploy, 'mu/deploy'
   autoload :MommaCat, 'mu/mommacat'
@@ -652,7 +646,7 @@ module MU
     new_cfg = $MU_CFG.dup
     examples = {}
     MU::Cloud.supportedClouds.each { |cloud|
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+      cloudclass = MU::Cloud.cloudClass(cloud)
       begin
         if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
           cfg_blob = cloudclass.hosted_config
@@ -808,11 +802,7 @@ module MU
   # @param groomer [String]: The grooming agent to load.
   # @return [Class]: The class object implementing this groomer agent
   def self.loadGroomer(groomer)
-    if !File.size?(MU.myRoot+"/modules/mu/groomers/#{groomer.downcase}.rb")
-      raise MuError, "Requested to use unsupported grooming agent #{groomer}"
-    end
-    require "mu/groomers/#{groomer.downcase}"
-    return Object.const_get("MU").const_get("Groomer").const_get(groomer)
+    MU::Groomer.loadGroomer(groomer)
   end
 
   @@myRegion_var = nil
@@ -966,8 +956,7 @@ module MU
 
   @@myCloudDescriptor = nil
   if MU.myCloud
-    svrclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud).const_get("Server")
-    found = svrclass.find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
+    found = MU::Cloud.resourceClass(MU.myCloud, "Server").find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
 #    found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
     if !found.nil? and found.size == 1
       @@myCloudDescriptor = found.values.first
@@ -980,8 +969,7 @@ module MU
   def self.myVPCObj
     return nil if MU.myCloud.nil?
     return @@myVPCObj_var if @@myVPCObj_var
-    cloudclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud)
-    @@myVPCObj_var ||= cloudclass.myVPCObj
+    @@myVPCObj_var ||= MU::Cloud.cloudClass(MU.myCloud).myVPCObj
     @@myVPCObj_var
   end
 
@@ -1106,10 +1094,9 @@ module MU
 
     clouds = platform.nil? ? MU::Cloud.supportedClouds : [platform]
     clouds.each { |cloud|
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-      bucketname = cloudclass.adminBucketName(credentials)
+      bucketname = MU::Cloud.cloudClass(cloud).adminBucketName(credentials)
       begin
-        if platform or (cloudclass.hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
+        if platform or (MU::Cloud.cloudClass(cloud).hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
           return bucketname
         end
       end

data/modules/mu/adoption.rb

@@ -30,7 +30,7 @@ module MU
       :omnibus => "Jam everything into one monolothic configuration"
     }
 
-    def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false)
+    def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false, pattern: nil)
       @scraped = {}
       @clouds = clouds
       @types = types
@@ -44,8 +44,11 @@ module MU
       @savedeploys = savedeploys
       @diff = diff
       @habitats = habitats
+      @regions = regions
       @habitats ||= []
       @scrub_mu_isms = scrub_mu_isms
+      @merge = merge
+      @pattern = pattern
     end
 
     # Walk cloud providers with available credentials to discover resources
@@ -53,7 +56,7 @@ module MU
       @default_parent = nil
 
       @clouds.each { |cloud|
-        cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+        cloudclass = MU::Cloud.cloudClass(cloud)
         next if cloudclass.listCredentials.nil?
 
         if cloud == "Google" and !@parent and @target_creds
@@ -65,7 +68,6 @@ module MU
 
         cloudclass.listCredentials.each { |credset|
           next if @sources and !@sources.include?(credset)
-
           cfg = cloudclass.credConfig(credset)
           if cfg and cfg['restrict_to_habitats']
             cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
@@ -90,7 +92,7 @@ module MU
 
           @types.each { |type|
             begin
-              resclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
+              resclass = MU::Cloud.resourceClass(cloud, type)
             rescue ::MU::Cloud::MuCloudResourceNotImplemented
               next
             end
@@ -106,6 +108,7 @@ module MU
               credentials: credset,
               allow_multi: true,
               habitats: @habitats.dup,
+              region: @regions,
               dummy_ok: true,
               skip_provider_owned: true,
 #              debug: false#,
@@ -114,7 +117,9 @@ module MU
 
             if found and found.size > 0
               if resclass.cfg_plural == "habitats"
-                found.reject! { |h| !cloudclass.listHabitats(credset).include?(h) }
+                found.reject! { |h|
+                  !cloudclass.listHabitats(credset).include?(h.cloud_id)
+                }
               end
               MU.log "Found #{found.size.to_s} raw #{resclass.cfg_plural} in #{cloud}"
               @scraped[type] ||= {}
@@ -122,7 +127,12 @@ module MU
                 if obj.habitat and !cloudclass.listHabitats(credset).include?(obj.habitat)
                   next
                 end
+
                 # XXX apply any filters (e.g. MU-ID tags)
+                if obj.cloud_id.nil?
+                  MU.log "This damn thing gave me no cloud id, what do I even do with that", MU::ERR, details: obj
+                  exit
+                end
                 @scraped[type][obj.cloud_id] = obj
               }
             end
@@ -200,7 +210,36 @@ module MU
         prefix = "mu" if prefix.empty? # so that appnames aren't ever empty
       end
 
+      # Find any previous deploys with this particular profile, which we'll use
+      # later for --diff.
+      @existing_deploys = {}
+      @existing_deploys_by_id = {}
+      @origins = {}
+      @types_found_in = {}
+      groupings.each_pair { |appname, types|
+        allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
+        next if (types & allowed_types).size == 0
+        origin = {
+          "appname" => prefix+appname,
+          "types" => (types & allowed_types).sort,
+          "habitats" => @habitats.sort,
+          "group_by" => @group_by.to_s
+        }
+
+        @existing_deploys[appname] = MU::MommaCat.findMatchingDeploy(origin)
+        if @existing_deploys[appname]
+          @existing_deploys_by_id[@existing_deploys[appname].deploy_id] = @existing_deploys[appname]
+          @origins[appname] = origin
+          origin['types'].each { |t|
+            @types_found_in[t] = @existing_deploys[appname]
+          }
+        end
+      }
+
       groupings.each_pair { |appname, types|
+        allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
+        next if (types & allowed_types).size == 0
+
         bok = { "appname" => prefix+appname }
         if @scrub_mu_isms
           bok["scrub_mu_isms"] = true
@@ -210,26 +249,23 @@ module MU
         end
 
         count = 0
-        allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
-        next if (types & allowed_types).size == 0
-        origin = {
-          "appname" => bok['appname'],
-          "types" => (types & allowed_types).sort,
-          "habitats" => @habitats.sort,
-          "group_by" => @group_by.to_s
-        }
-
-        deploy = MU::MommaCat.findMatchingDeploy(origin)
-        if @diff and !deploy
-          MU.log "--diff was set but I failed to find a deploy like me to compare to", MU::ERR, details: origin
-          exit 1
+        if @diff
+          if !@existing_deploys[appname]
+            MU.log "--diff was set but I failed to find a deploy like '#{appname}' to compare to (have #{@existing_deploys.keys.join(", ")})", MU::ERR, details: @origins[appname]
+            exit 1
+          else
+            MU.log "Will diff current live resources against #{@existing_deploys[appname].deploy_id}", MU::NOTICE, details: @origins[appname]
+          end
         end
 
         threads = []
+        timers = {}
+        walltimers = {}
         @clouds.each { |cloud|
           @scraped.each_pair { |type, resources|
+            typestart = Time.now
             res_class = begin
-              MU::Cloud.loadCloudType(cloud, type)
+              MU::Cloud.resourceClass(cloud, type)
             rescue MU::Cloud::MuCloudResourceNotImplemented
               # XXX I don't think this can actually happen
               next
@@ -237,6 +273,7 @@ module MU
             next if !types.include?(res_class.cfg_plural)
 
             bok[res_class.cfg_plural] ||= []
+            timers[type] ||= {}
 
             class_semaphore = Mutex.new
 
@@ -253,13 +290,18 @@ module MU
                 end
               end
               threads << Thread.new(obj_thr) { |obj|
+                start = Time.now
 
-                kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats)
-                if kitten_cfg
+                kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats, types: @types)
+                if kitten_cfg and (!@pattern or @pattern.match(kitten_cfg['name']))
                   print "."
                   kitten_cfg.delete("credentials") if @target_creds
                   class_semaphore.synchronize {
                     bok[res_class.cfg_plural] << kitten_cfg
+                    if !kitten_cfg['cloud_id']
+                      MU.log "No cloud id in this #{res_class.cfg_name} kitten!", MU::ERR, details: kitten_cfg
+                    end
+                    timers[type][kitten_cfg['cloud_id']] = (Time.now - start)
                   }
                   count += 1
                 end
@@ -270,6 +312,7 @@ module MU
             threads.each { |t|
               t.join
             }
+
             puts ""
             bok[res_class.cfg_plural].sort! { |a, b|
               strs = [a, b].map { |x|
@@ -291,24 +334,30 @@ module MU
               bok[res_class.cfg_plural].each { |sibling|
                 next if kitten_cfg == sibling
                 if sibling['name'] == kitten_cfg['name']
-                  MU.log "#{res_class.cfg_name} name #{sibling['name']} unavailable, will attempt to rename duplicate object", MU::DEBUG, details: kitten_cfg
-                  if kitten_cfg['parent'] and kitten_cfg['parent'].respond_to?(:id) and kitten_cfg['parent'].id
-                    kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['parent'].id
-                  elsif kitten_cfg['project']
-                    kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['project']
-                  elsif kitten_cfg['region']
-                    kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['region']
-                  elsif kitten_cfg['cloud_id']
-                    kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['cloud_id'].gsub(/[^a-z0-9]/i, "-")
-                  else
-                    raise MU::Config::DuplicateNameError, "Saw duplicate #{res_class.cfg_name} name #{sibling['name']} and couldn't come up with a good way to differentiate them"
-                  end
+                  MU::Adoption.deDuplicateName(kitten_cfg, res_class)
                   MU.log "De-duplication: Renamed #{res_class.cfg_name} name '#{sibling['name']}' => '#{kitten_cfg['name']}'", MU::NOTICE
                   break
                 end
               }
             }
+            walltimers[type] ||= 0
+            walltimers[type] += (Time.now - typestart)
+          }
+        }
+
+        timers.each_pair { |type, resources|
+          next if resources.empty?
+          total = resources.values.sum
+          top_5 =  resources.keys.sort { |a, b|
+            resources[b] <=> resources[a]
+          }.slice(0, 5).map { |k|
+            k.to_s+": "+sprintf("%.2fs", resources[k])
           }
+          if walltimers[type] < 45
+            MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])}"
+          else
+            MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])} (CPU time #{sprintf("%.2fs", total)}, avg #{sprintf("%.2fs", total/resources.size)}). Top 5:", MU::NOTICE, details: top_5
+          end
         }
 
         # No matching resources isn't necessarily an error
@@ -317,23 +366,36 @@ module MU
 # Now walk through all of the Refs in these objects, resolve them, and minimize
 # their config footprint
         MU.log "Minimizing footprint of #{count.to_s} found resources", MU::DEBUG
-        @boks[bok['appname']] = vacuum(bok, origin: origin, save: @savedeploys)
 
-        if @diff and !deploy
+        generated_deploy = generateStubDeploy(bok)
+        @boks[bok['appname']] = vacuum(bok, origin: @origins[appname], deploy: generated_deploy, save: @savedeploys)
+
+        if @diff and !@existing_deploys[appname]
           MU.log "diff flag set, but no comparable deploy provided for #{bok['appname']}", MU::ERR
           exit 1
         end
 
-        if deploy and @diff
-          prevcfg = MU::Config.manxify(vacuum(deploy.original_config, deploy: deploy))
+        if @diff
+          prev_vacuumed = vacuum(@existing_deploys[appname].original_config, deploy: @existing_deploys[appname], keep_missing: true, copy_from: generated_deploy)
+          prevcfg = MU::Config.manxify(prev_vacuumed)
           if !prevcfg
-            MU.log "#{deploy.deploy_id} didn't have a working original config for me to compare", MU::ERR
+            MU.log "#{@existing_deploys[appname].deploy_id} didn't have a working original config for me to compare", MU::ERR
             exit 1
           end
           newcfg = MU::Config.manxify(@boks[bok['appname']])
+          report = prevcfg.diff(newcfg)
+
+          if report
+
+            if MU.muCfg['adopt_change_notify']
+              notifyChanges(@existing_deploys[appname], report.freeze)
+            end
+            if @merge
+              MU.log "Saving changes to #{@existing_deploys[appname].deploy_id}"
+              @existing_deploys[appname].updateBasketofKittens(newcfg, save_now: true)
+            end
+          end
 
-          prevcfg.diff(newcfg)
-          exit
         end
       }
       @boks
@@ -341,6 +403,183 @@ module MU
 
     private
 
+    # @param tier [Hash]
+    # @param parent_key [String]
+    def crawlChangeReport(tier, parent_key = nil, indent: "")
+      report = []
+      if tier.is_a?(Array)
+        tier.each { |a|
+          sub_report = crawlChangeReport(a, parent_key)
+          report.concat(sub_report) if sub_report and !sub_report.empty?
+        }
+      elsif tier.is_a?(Hash)
+        if tier[:action]
+          preposition = if tier[:action] == :added
+            "to"
+          elsif tier[:action] == :removed
+            "from"
+          else
+            "in"
+          end
+
+          name = ""
+          type_of = parent_key.sub(/s$|\[.*/, '') if parent_key
+          loc = tier[:habitat]
+
+          if tier[:value] and tier[:value].is_a?(Hash)
+            name, loc = MU::MommaCat.getChunkName(tier[:value], type_of)
+          elsif parent_key
+            name = parent_key
+          end
+
+          path_str = []
+          slack_path_str = ""
+          if tier[:parents] and tier[:parents].size > 2
+            path = tier[:parents].clone
+            slack_path_str += "#{preposition} \*"+path.join(" ⇨ ")+"\*" if path.size > 0
+            path.shift
+            path.shift
+            path.pop if path.last == name
+            for c in (0..(path.size-1)) do
+              path_str << ("  " * (c+2)) + (path[c] || "<nil>")
+            end
+          end
+          path_str << "" if !path_str.empty?
+
+          plain = (name ? name : type_of) if name or type_of
+          plain ||= "" # XXX but this is a problem
+          slack = "`"+plain+"`"
+
+          plain += " ("+loc+")" if loc and !loc.empty?
+          color = plain
+
+          if tier[:action] == :added
+            color = "+ ".green + plain
+            plain = "+ " + plain
+            slack += " added"
+          elsif tier[:action] == :removed
+            color = "- ".red + plain
+            plain = "- " + plain
+            slack += " removed"
+          end
+
+          slack += " #{tier[:action]} #{preposition} \*#{loc}\*" if loc and !loc.empty? and [Array, Hash].include?(tier[:value].class)
+
+          plain = path_str.join(" => \n") + indent + plain
+          color = path_str.join(" => \n") + indent + color
+
+          slack += " "+slack_path_str if !slack_path_str.empty?
+          myreport = {
+            "slack" => slack,
+            "plain" => plain,
+            "color" => color
+          }
+
+          append = ""
+          if tier[:value] and (tier[:value].is_a?(Array) or tier[:value].is_a?(Hash))
+            if tier[:value].is_a?(Hash)
+              if name
+                tier[:value].delete("entity")
+                tier[:value].delete(name.sub(/\[.*/, '')) if name
+              end
+              if (tier[:value].keys - ["id", "name", "type"]).size > 0
+                myreport["details"] = tier[:value].clone
+                append = PP.pp(tier[:value], '').gsub(/(^|\n)/, '\1'+indent)
+              end
+            else
+              append = indent+"["+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s.light_blue }.join(", ")+"]"
+              slack += " #{tier[:action].to_s}: "+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s }.join(", ")
+            end
+          else
+            tier[:value] ||= "<nil>"
+            if ![:removed].include?(tier[:action])
+              myreport["slack"] += ". New #{tier[:field] ? "`"+tier[:field]+"`" : :value}: \*#{tier[:value]}\*"
+            else
+              myreport["slack"] += " (was \*#{tier[:value]}\*)"
+            end
+            append = tier[:value].to_s.bold
+          end
+
+          if append and !append.empty?
+            myreport["plain"] += " =>\n  "+indent+append
+            myreport["color"] += " =>\n  "+indent+append
+          end
+
+          report << myreport if tier[:action]
+        end
+
+        # Just because we've got changes at this level doesn't mean there aren't
+        # more further down.
+        tier.each_pair { |k, v|
+          next if !(v.is_a?(Hash) or v.is_a?(Array))
+          sub_report = crawlChangeReport(v, k, indent: indent+"  ")
+          report.concat(sub_report) if sub_report and !sub_report.empty?
+        }
+      end
+
+      report
+    end
+
+
+    def notifyChanges(deploy, report)
+      snippet_threshold = (MU.muCfg['adopt_change_notify'] && MU.muCfg['adopt_change_notify']['slack_snippet_threshold']) || 5
+
+      report.each_pair { |res_type, resources|
+        shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(res_type, false)
+        next if !shortclass # we don't really care about Mu metadata changes
+        resources.each_pair { |name, data|
+          if MU::MommaCat.getChunkName(data[:value], res_type).first.nil?
+            symbol = if data[:action] == :added
+              "+".green
+            elsif data[:action] == :removed
+              "-".red
+            else
+              "~".yellow
+            end
+            puts (symbol+" "+res_type+"["+name+"]")
+          end
+
+          noun = shortclass ? shortclass.to_s : res_type.capitalize
+          verb = if data[:action]
+            data[:action].to_s
+          else
+            "modified"
+          end
+
+          changes = crawlChangeReport(data.freeze, res_type)
+
+          slacktext = "#{noun} \*#{name}\* was #{verb}"
+          if data[:habitat]
+            slacktext += " in \*#{data[:habitat]}\*"
+          end
+          snippets = []
+
+          if [:added, :removed].include?(data[:action]) and data[:value]
+            snippets << { text: "```"+JSON.pretty_generate(data[:value])+"```" }
+          else
+            changes.each { |c|
+              slacktext += "\n • "+c["slack"]
+              if c["details"]
+                details = JSON.pretty_generate(c["details"])
+                snippets << { text: "```"+JSON.pretty_generate(c["details"])+"```" }
+              end
+            }
+          end
+
+          changes.each { |c|
+            puts c["color"]
+          }
+          puts ""
+
+          if MU.muCfg['adopt_change_notify'] and MU.muCfg['adopt_change_notify']['slack']
+            deploy.sendAdminSlack(slacktext, scrub_mu_isms: MU.muCfg['adopt_scrub_mu_isms'], snippets: snippets, noop: false)
+          end
+
+        }
+      }
+
+    end
+
     def scrubSchemaDefaults(conf_chunk, schema_chunk, depth = 0, type: nil)
       return if schema_chunk.nil?
 
@@ -372,8 +611,7 @@ module MU
           # theory
           realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])
 
-            cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type)
-            _toplevel_required, cloudschema = cloudclass.schema(self)
+            _toplevel_required, cloudschema = MU::Cloud.resourceClass(item['cloud'], type).schema(self)
 
             newschema = schema_chunk["items"].dup
             newschema["properties"].merge!(cloudschema)
@@ -397,8 +635,7 @@ module MU
     # Do the same for our main objects: if they all use the same credentials,
     # for example, remove the explicit +credentials+ attributes and set that
     # value globally, once.
-    def vacuum(bok, origin: nil, save: false, deploy: nil)
-      deploy ||= generateStubDeploy(bok)
+    def vacuum(bok, origin: nil, save: false, deploy: nil, copy_from: nil, keep_missing: false)
 
       globals = {
         'cloud' => {},
@@ -418,11 +655,24 @@ module MU
               end
             }
             obj = deploy.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
+            inject_metadata = save
+            if obj.nil? and copy_from
+              obj = copy_from.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
+              if obj
+                inject_metadata = true
+                obj.intoDeploy(deploy, force: true)
+              end
+            end
+
             begin
               raise Incomplete if obj.nil?
+              if inject_metadata
+                deploydata = obj.notify
+                deploy.notify(attrs[:cfg_plural], resource['name'], deploydata, triggering_node: obj)
+              end
               new_cfg = resolveReferences(resource, deploy, obj)
               new_cfg.delete("cloud_id")
-              cred_cfg = MU::Cloud.const_get(obj.cloud).credConfig(obj.credentials)
+              cred_cfg = MU::Cloud.cloudClass(obj.cloud).credConfig(obj.credentials)
               if cred_cfg['region'] == new_cfg['region']
                 new_cfg.delete('region')
               end
@@ -432,6 +682,11 @@ module MU
               end
               processed << new_cfg
             rescue Incomplete
+              if keep_missing
+                processed << resource
+              else
+                MU.log "#{attrs[:cfg_name]} #{resource['name']} didn't show up from findLitterMate", MU::WARN, details: deploy.original_config[attrs[:cfg_plural]].reject { |r| r['name'] != "" }
+              end
             end
           }
 
@@ -442,24 +697,23 @@ module MU
 
       # Pare out global values like +cloud+ or +region+ that appear to be
       # universal in the deploy we're creating.
-      def scrub_globals(h, field)
+      scrub_globals = Proc.new { |h, field|
         if h.is_a?(Hash)
           newhash = {}
           h.each_pair { |k, v|
             next if k == field
-            newhash[k] = scrub_globals(v, field)
+            newhash[k] = scrub_globals.call(v, field)
           }
           h = newhash
         elsif h.is_a?(Array)
           newarr = []
           h.each { |v|
-            newarr << scrub_globals(v, field)
+            newarr << scrub_globals.call(v, field)
           }
-          h = newarr
+          h = newarr.uniq
         end
-
         h
-      end
+      }
 
       globals.each_pair { |field, counts|
         next if counts.size != 1
@@ -469,7 +723,7 @@ module MU
           if bok[attrs[:cfg_plural]]
             new_resources = []
             bok[attrs[:cfg_plural]].each { |resource|
-              new_resources << scrub_globals(resource, field)
+              new_resources << scrub_globals.call(resource, field)
             }
             bok[attrs[:cfg_plural]] = new_resources
           end
@@ -487,11 +741,33 @@ module MU
     end
 
     def resolveReferences(cfg, deploy, parent)
+      mask_deploy_id = false
+
+      check_deploy_id = Proc.new { |cfgblob|
+        (deploy and
+         (cfgblob.is_a?(MU::Config::Ref) or cfgblob.is_a?(Hash)) and
+         cfgblob['deploy_id'] and
+         cfgblob['deploy_id'] != deploy.deploy_id and
+         @diff and
+         @types_found_in[cfgblob['type']] and
+         @types_found_in[cfgblob['type']].deploy_id == cfgblob['deploy_id']
+        )
+      }
+
+      mask_deploy_id = check_deploy_id.call(cfg)
+
       if cfg.is_a?(MU::Config::Ref)
-        cfg.kitten(deploy) || cfg.kitten
+        if mask_deploy_id
+          cfg.delete("deploy_id")
+          cfg.delete("mommacat")
+          cfg.kitten(deploy)
+        else
+          cfg.kitten(deploy) || cfg.kitten
+        end
+
         hashcfg = cfg.to_h
 
-        if cfg.kitten(deploy)
+        if cfg.kitten
           littermate = deploy.findLitterMate(type: cfg.type, name: cfg.name, cloud_id: cfg.id, habitat: cfg.habitat)
 
           if littermate and littermate.config['name']
@@ -516,13 +792,12 @@ module MU
         elsif hashcfg["id"] and !hashcfg["name"]
           hashcfg.delete("deploy_id")
         else
-          pp parent.cloud_desc
-          raise Incomplete, "Failed to resolve reference on behalf of #{parent}"
+          raise Incomplete.new "Failed to resolve reference on behalf of #{parent}", details: hashcfg
         end
         hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id
 
         if parent and parent.config
-          cred_cfg = MU::Cloud.const_get(parent.cloud).credConfig(parent.credentials)
+          cred_cfg = MU::Cloud.cloudClass(parent.cloud).credConfig(parent.credentials)
 
           if parent.config['region'] == hashcfg['region'] or
              cred_cfg['region'] == hashcfg['region']
@@ -581,7 +856,12 @@ module MU
             MU.log "Dropping unresolved value", MU::WARN, details: value
           end
         }
-        cfg = new_array
+        cfg = new_array.uniq
+      end
+
+      if mask_deploy_id or check_deploy_id.call(cfg)
+        cfg.delete("deploy_id")
+        MU.log "#{parent} in #{deploy.deploy_id} references something in #{@types_found_in[cfg['type']].deploy_id}, ditching extraneous deploy_id", MU::DEBUG, details: cfg.to_h
       end
 
       cfg
@@ -631,6 +911,10 @@ module MU
 
             if !@scraped[typename][kitten['cloud_id']]
               MU.log "No object in scraped tree for #{attrs[:cfg_name]} #{kitten['cloud_id']} (#{kitten['name']})", MU::ERR, details: kitten
+              if kitten['cloud_id'].nil?
+                pp caller
+                exit
+              end
               next
             end
 
@@ -641,7 +925,8 @@ module MU
             deploy.addKitten(
               attrs[:cfg_plural],
               kitten['name'],
-              @scraped[typename][kitten['cloud_id']]
+              @scraped[typename][kitten['cloud_id']],
+              do_notify: true
             )
           }
         end
@@ -650,6 +935,21 @@ module MU
       deploy
     end
 
+    def self.deDuplicateName(kitten_cfg, res_class)
+      orig_name = kitten_cfg['name'].dup
+      if kitten_cfg['parent'] and kitten_cfg['parent'].respond_to?(:id) and kitten_cfg['parent'].id
+        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['parent'].id
+      elsif kitten_cfg['project']
+        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['project']
+      elsif kitten_cfg['region']
+        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['region']
+      elsif kitten_cfg['cloud_id']
+        kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['cloud_id'].gsub(/[^a-z0-9]/i, "-")
+      else
+        raise MU::Config::DuplicateNameError, "Saw duplicate #{res_class.cfg_name} name #{orig_name} and couldn't come up with a good way to differentiate them"
+      end
+    end
+
     # Go through everything we've scraped and update our mappings of cloud ids
     # and bare name fields, so that resources can reference one another
     # portably by name.