cloud-mu 3.1.6 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 447346eba4a1cd7ee0df18c2c85aea32d1a45af0f4d22474fc902007d1f30a2c
-  data.tar.gz: 8f44c2cf180c0748c712b9c244d0c21335147c6a3c9f6a1472772546e13a9b85
+  metadata.gz: fd9a15a0e94a578919c0cb22bf5c95ee86b5b4a03fcc0ea1c35da2c161e8cf31
+  data.tar.gz: 04cdc9d9a70de97fdaf12021166b707921ec981d7ab2dc92cd1ebce491a9e130
 SHA512:
-  metadata.gz: 8008e86471d5596337e3b642f5740a2bfe3b178646dd36a37f23dcfb8e0eacfcfbab4bac148ec5855f74f803543fd333480636249ec938387523e6f0c1fddde8
-  data.tar.gz: '08466a848ca7b54fc6460e240f472cc812e93730650545f240a92ea67bde875b96a5d8398b770182355721c6628887665f938dcb7a3103209f7e170cffa246b9'
+  metadata.gz: adfef4f231a946a3929b7f8026a9e338658dadfef3482942a46e8fdb9b10c4ca7d68aa457d135815a3878882d88a103ec1ebacea4bcb1ae4c45b070d85ceee06
+  data.tar.gz: 3a7b04d7d2486b05e8d413b78029b610bb7ba6bd8a33a1c24cf7b8bdf63dabaf9e0a1baab8408a66f995d05df719261248cbf27ee5b230a350109053ddb0420a

data/bin/mu-adopt

@@ -21,12 +21,6 @@ require 'bundler/setup'
 require 'optimist'
 require 'mu'
 
-available_clouds = MU::Cloud.supportedClouds
-available_clouds.reject! { |cloud|
-  cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-  cloudclass.listCredentials.nil? or cloudclass.listCredentials.size == 0
-}
-
 available_types = MU::Cloud.resource_types.keys.map { |t| t.to_s }
 grouping_options = {
   "logical" => "Group resources in logical layers (folders and habitats together, users/roles/groups together, network resources together, etc)",
@@ -39,15 +33,17 @@ $opt = Optimist::options do
   EOS
   opt :appname, "The overarching name of the application stack we will generate", :required => false, :default => "mu", :type => :string
   opt :types, "The resource types to scan and import. Valid types: #{available_types.join(", ")}", :required => false, :type => :strings, :default => available_types
-  opt :clouds, "The cloud providers to scan and import.", :required => false, :type => :strings, :default => available_clouds
+  opt :clouds, "The cloud providers to scan and import.", :required => false, :type => :strings, :default => MU::Cloud.availableClouds
   opt :parent, "Where applicable, resources which reside in the root folder or organization are configured with the specified parent in our target BoK", :required => false, :type => :string
   opt :billing, "Force-set this billing entity on created resources, instead of copying from the live resources", :required => false, :type => :string
   opt :sources, "One or more sets of credentials to use when importing resources. By default we will search and import from all sets of available credentials for each cloud provider specified with --clouds", :required => false, :type => :strings
   opt :credentials, "Override the 'credentials' value in our generated Baskets of Kittens to target a single, specific account. Our default behavior is to set each resource to deploy into the account from which it was sourced.", :required => false, :type => :string
   opt :savedeploys, "Generate actual deployment metadata in #{MU.dataDir}/deployments, as though the resources we found were created with mu-deploy. If we are generating more than one configuration, and a resource needs to reference another resource (e.g. to declare a VPC in which to reside), this will allow us to reference them as virtual resource, rather than by raw cloud identifier.", :required => false, :type => :boolean, :default => false
   opt :diff, "List the differences between what we find and an existing, saved deploy from a previous run, if one exists.", :required => false, :type => :boolean
+  opt :merge_changes, "When using --diff, merge detected changes into the baseline deploy after reporting on them.", :required => false, :type => :boolean, :default => false
   opt :grouping, "Methods for grouping found resources into separate Baskets.\n\n"+MU::Adoption::GROUPMODES.keys.map { |g| "* "+g.to_s+": "+MU::Adoption::GROUPMODES[g] }.join("\n")+"\n\n", :required => false, :type => :string, :default => "logical"
   opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
+  opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
   opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false
 end
 
@@ -102,8 +98,7 @@ if !ok
   exit 1
 end
 
-
-adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub])
+adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes])
 found = adoption.scrapeClouds
 if found.nil? or found.empty?
   MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }
@@ -117,10 +112,7 @@ boks.each_pair { |appname, bok|
   File.open("#{appname}.yaml", "w") { |f|
     f.write JSON.parse(JSON.generate(bok)).to_yaml
   }
-  conf_engine = MU::Config.new("#{appname}.yaml")
-  stack_conf = conf_engine.config
 #  puts stack_conf.to_yaml
-  MU.log "#{appname}.yaml validated successfully", MU::NOTICE
   MU::Cloud.resource_types.each_pair { |type, cfg|
     if bok[cfg[:cfg_plural]]
       MU.log "#{bok[cfg[:cfg_plural]].size.to_s} #{cfg[:cfg_plural]}", MU::NOTICE

data/bin/mu-azure-tests

@@ -0,0 +1,57 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+(0..100000).to_a.each { |n|
+retries = 0
+seed = nil
+#        begin
+#          raise MuError, "Failed to allocate an unused MU-ID after #{retries} tries!" if retries > 70
+#          seedsize = 1 + (retries/10).abs
+#          seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
+#        end while seed == "mu" or seed[0] == seed[1]
+seed = "nn"
+handle = MU::MommaCat.generateHandle(seed)
+puts handle
+}
+exit
+
+#pp MU::Cloud::Azure.listRegions
+#pp MU::Cloud::Azure::Habitat.testcalls
+#pp MU::Cloud::Azure::VPC.find(cloud_id: MU::Cloud::Azure::Id.new(resource_group: "mu", name: "mu-vnet"))
+#pp MU::Cloud::Azure.authorization.role_assignments.list_for_resource_group("AKS-DEV-2019062015-KA-EASTUS")
+#pp MU::Cloud::Azure::Role.find(role_name: "Azure Kubernetes Service Cluster Admin Role")
+#puts MU::Cloud::Azure.default_subscription
+#pp MU::Cloud::Azure.fetchPublicIP("MYVPC-DEV-2019061911-XI-EASTUS", "ip-addr-thingy")
+#pp MU::Cloud::Azure.ensureProvider("egtazure", "Microsoft.ContainerService", force: true)
+pp MU::Cloud::Azure::Server.find(cloud_id: "mu")
+exit
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/6")
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/8")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/6")
+pp MU::Cloud::Azure::Server.fetchImage("Debian/debian-10/10")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2012-R2-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2016-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2019-Datacenter")

data/bin/mu-cleanup

@@ -24,10 +24,8 @@ require 'mu'
 Dir.chdir(MU.installDir)
 
 credentials = []
-MU::Cloud.supportedClouds.each { |cloud|
-  cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-  next if cloudclass.listCredentials.nil? or cloudclass.listCredentials.size == 0
-  credentials.concat(cloudclass.listCredentials)
+MU::Cloud.availableClouds.each { |cloud|
+  credentials.concat(MU::Cloud.cloudClass(cloud).listCredentials)
 }
 credentials.uniq!
 

data/bin/mu-configure

@@ -113,12 +113,44 @@ $CONFIGURABLES = {
     "desc" => "Disable the Momma Cat grooming daemon. Nodes which require asynchronous Ansible/Chef bootstraps will not function. This option is only honored in gem-based installations.",
     "boolean" => true
   },
+  "adopt_change_notify" => {
+    "title" => "Adoption Change Notifications",
+    "subtree" => {
+      "slack" => {
+        "title" => "Send to Slack",
+        "desc" => "Report modifications to adopted resources, detected by mu-adopt --diff, to the Slack webhook and channel configured under Slack Configuration.",
+        "boolean" => true
+      },
+      "slack_snippet_threshold" => {
+        "title" => "Attachment Threshold",
+        "desc" => "If a list of details about a modified resources is longer than this number of lines (in JSON), it will be sent as an \"attachment,\" which in Slack means a blockquote that displays a few lines with a \"Show more\" button. The internal default is 5 lines."
+      },
+#      "email" => {
+#        "title" => "Send Email",
+#        "desc" => "",
+#        "boolean" => true
+#      }
+    }
+  },
   "adopt_scrub_mu_isms" => {
-    "title" => "Disable Momma Cat",
+    "title" => "Scrub Mu-isms from Baskets of Kittens",
     "default" => false,
     "desc" => "Ordinarily, Mu will automatically name, tag and generate auxiliary resources in a standard Mu-ish fashion that allows for deployment of multiple clones of a given stack. Toggling this flag will change the default behavior of mu-adopt, when it creates stack descriptors from found resources, to enable or disable this behavior (see also mu-adopt's --scrub option).",
     "boolean" => true
   },
+  "slack" => {
+    "title" => "Slack Configuration",
+    "subtree" => {
+      "webhook" => {
+        "title" => "Webhook",
+        "desc" => "The hooks.slack.com URL for the webook to which we'll send deploy notifications"
+      },
+      "channel" => {
+        "title" => "Channel",
+        "desc" => "The channel name (without leading #) to which alerts should be sent."
+      }
+    }
+  },
   "mommacat_port" => {
     "title" => "Momma Cat Listen Port",
     "pattern" => /^[0-9]+$/i,
@@ -247,6 +279,10 @@ $CONFIGURABLES = {
         "required" => false,
         "desc" => "For Google Cloud projects which are attached to a GSuite domain. GCP service accounts cannot view or manage GSuite resources (groups, users, etc) directly, but must instead masquerade as a GSuite user which has delegated authority to the service account. See also: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority"
       },
+      "org" => {
+        "title" => "Default Org/Domain",
+        "desc" => "For credential sets which have access to multiple GSuite or Cloud Identity orgs, you must specify a default organization (e.g. my.domain.com)."
+      },
       "customer_id" => {
         "title" => "GSuite Customer ID",
         "required" => false,

data/bin/mu-deploy

@@ -105,7 +105,7 @@ if $opts[:dryrun]
     Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :never) {
       begin
         Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :immediate) {
-          MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
+          MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
           Thread.current.exit
         }
       ensure
@@ -124,7 +124,7 @@ if $opts[:dryrun]
       )
       cost_dummy_deploy.run
     rescue MU::Cloud::MuCloudResourceNotImplemented, MU::Cloud::MuCloudFlagNotImplemented
-      MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
+      MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
     end
   end
   exit
@@ -135,7 +135,7 @@ if $opts[:update]
 # TODO consider whether this is useful/valid
 #  old_conf = JSON.parse(File.read(deploy.deploy_dir+"/basket_of_kittens.json"))
 #  stack_conf = old_conf.merge(stack_conf)
-  deploy.updateBasketofKittens(stack_conf)
+  deploy.updateBasketofKittens(stack_conf, skip_validation: true)
   deployer = MU::Deploy.new(
     deploy.environment,
     verbosity: verbosity,

data/bin/mu-findstray-tests

@@ -0,0 +1,25 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, debug: true)

data/bin/mu-gen-docs

@@ -79,8 +79,7 @@ EOF
       impl_counts[type] ||= 0
       [a, b].each { |cloud|
         begin
-          myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
-          case myclass.quality
+          case MU::Cloud.resourceClass(cloud, type).quality
           when MU::Cloud::RELEASE
             cloud_is_useful[cloud] = true
             counts[cloud] += 4
@@ -114,8 +113,7 @@ EOF
     cloudlist.each { |cloud|
       readme += "<td><center>"
       begin
-        myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
-        case myclass.quality
+        case MU::Cloud.resourceClass(cloud, type).quality
         when MU::Cloud::RELEASE
           readme += "<img src='release.png' style='#{icon_style}' title='Release Quality' alt='[Release Quality]'>"
         when MU::Cloud::BETA

data/bin/mu-run-tests

@@ -42,7 +42,7 @@ only = ARGV
 
 files = Dir.glob("*.yaml", base: dir)
 files.concat(Dir.glob("*.yml", base: dir))
-baseclouds = MU::Cloud.supportedClouds.reject { |c| c == "CloudFormation" }
+baseclouds = MU::Cloud.availableClouds.reject { |c| c == "CloudFormation" }
 
 commands = {}
 failures = []
@@ -56,20 +56,33 @@ end
 
 files.each { |f|
   clouds = baseclouds.dup
+  groomer_match = true
   File.open(dir+"/"+f).readlines.each { |l|
     l.chomp!
-    next if !l.match(/^\s*#\s*clouds: (.*)/)
-    clouds = []
-    cloudstr = Regexp.last_match[1]
-    cloudstr.split(/\s*,\s*/).each { |c|
-      baseclouds.each { |cloud|
-        if cloud.match(/^#{Regexp.quote(c)}$/i)
-          clouds << cloud
+    if l.match(/^\s*#\s*clouds: (.*)/)
+      clouds = []
+      cloudstr = Regexp.last_match[1]
+      cloudstr.split(/\s*,\s*/).each { |c|
+        baseclouds.each { |cloud|
+          if cloud.match(/^#{Regexp.quote(c)}$/i)
+            clouds << cloud
+          end
+        }
+      }
+    elsif l.match(/^\s*#\s*groomers: (.*)/)
+      groomerstr = Regexp.last_match[1]
+      groomerstr.split(/\s*,\s*/).each { |g|
+        if !MU::Groomer.availableGroomers.include?(g)
+          MU.log "#{f} requires groomer #{g}, which is not available. This test will be skipped.", MU::NOTICE
+          groomer_match = false
         end
       }
-    }
-    break
+    end
   }
+  if !groomer_match
+    next
+  end
+
   clouds.each { |cloud|
     cmd = "mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"
     commands[cmd] = {

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.1.6'
-  s.date        = '2020-03-20'
+  s.version     = '3.2.0'
+  s.date        = '2020-06-16'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"

data/cookbooks/mu-tools/libraries/helper.rb

@@ -236,7 +236,7 @@ module Mutools
       response = nil
       begin
         secret = get_deploy_secret
-        if secret.nil?
+        if secret.nil? or secret.empty?
           raise "Failed to fetch deploy secret, and I can't communicate with Momma Cat without it"
         end
 

data/cookbooks/mu-tools/recipes/apply_security.rb

@@ -252,21 +252,21 @@ if !node['application_attributes']['skip_recipes'].include?('apply_security')
       #		end
       # 6.3 Configure PAM
       # 6.3.2 Set Password Creation Requirement Parameters Using pam_cracklib
-      template "/etc/pam.d/password-auth-local" do
-        source "etc_pamd_password-auth.erb"
-        mode 0644
-      end
-      link "/etc/pam.d/password-auth" do
-        to "/etc/pam.d/password-auth-local"
-      end
+#      template "/etc/pam.d/password-auth-local" do
+#        source "etc_pamd_password-auth.erb"
+#        mode 0644
+#      end
+#      link "/etc/pam.d/password-auth" do
+#        to "/etc/pam.d/password-auth-local"
+#      end
       #6.3.3 Set Lockout for Failed Password Attempts
-      template "/etc/pam.d/system-auth-local" do
-        source "etc_pamd_system-auth.erb"
-        mode 0644
-      end
-      link "/etc/pam.d/system-auth" do
-        to "/etc/pam.d/system-auth-local"
-      end
+#      template "/etc/pam.d/system-auth-local" do
+#        source "etc_pamd_system-auth.erb"
+#        mode 0644
+#      end
+#      link "/etc/pam.d/system-auth" do
+#        to "/etc/pam.d/system-auth-local"
+#      end
   
       #SV-50303r1_rule/SV-50304r1_rule
       execute "chown root:root /etc/shadow"

data/cookbooks/mu-tools/recipes/aws_api.rb

@@ -21,3 +21,12 @@ chef_gem "aws-sdk-core" do
   version "2.11.24"
   action :install
 end
+
+if platform_family?("rhel") or platform_family?("amazon")
+  if node['platform_version'].to_i == 6
+    package "python34-pip"
+    execute "/usr/bin/pip3 install awscli" do
+      not_if "test -x /usr/bin/aws"
+    end
+  end
+end

data/extras/generate-stock-images

@@ -91,6 +91,7 @@ $opts[:clouds].each { |cloud|
         end
         next if !needed
       end
+      MU.log "Loading "+bok_dir+"/"+cloud+"/"+platform+".yaml"
       conf_engine = MU::Config.new(
         bok_dir+"/"+cloud+"/"+platform+".yaml",
         default_credentials: $opts[(cloud.downcase+"_creds").to_sym]

data/modules/mu.rb

@@ -79,38 +79,40 @@ class Hash
     }
     return 0 if self == other # that was easy!
     # compare elements and decide who's "bigger" based on their totals?
-    0
+
+    # fine, try some brute force and just hope everything implements to_s
+    self.flatten.map { |e| e.to_s }.join() <=> other.flatten.map { |e| e.to_s }.join()
   end
 
-  # Recursively compare two hashes
-  def diff(with, on = self, level: 0, parents: [])
+  # Recursively compare two Mu Basket of Kittens hashes and report the differences
+  def diff(with, on = self, level: 0, parents: [], report: {}, habitat: nil)
     return if with.nil? and on.nil?
     if with.nil? or on.nil? or with.class != on.class
       return # XXX ...however we're flagging differences
     end
     return if on == with
 
-    tree = ""
-    indentsize = 0
-    parents.each { |p|
-      tree += (" " * indentsize) + p + " => \n"
-      indentsize += 2
-    }
-    indent = (" " * indentsize)
-
     changes = []
+    report ||= {}
     if on.is_a?(Hash)
       on_unique = (on.keys - with.keys)
       with_unique = (with.keys - on.keys)
       shared = (with.keys & on.keys)
       shared.each { |k|
-        diff(with[k], on[k], level: level+1, parents: parents + [k])
+
+        report_data = diff(with[k], on[k], level: level+1, parents: parents + [k], report: report[k], habitat: habitat)
+        if report_data and !report_data.empty?
+          report ||= {}
+          report[k] = report_data
+        end
       }
       on_unique.each { |k|
-        changes << "- ".red+PP.pp({k => on[k] }, '')
+        report[k] = { :action => :removed, :parents => parents, :value => on[k].clone }
+        report[k][:habitat] = habitat if habitat
       }
       with_unique.each { |k|
-        changes << "+ ".green+PP.pp({k => with[k]}, '')
+        report[k] = { :action => :added, :parents => parents, :value => with[k].clone }
+        report[k][:habitat] = habitat if habitat
       }
     elsif on.is_a?(Array)
       return if with == on
@@ -122,29 +124,27 @@ class Hash
       # sorting arrays full of weird, non-primitive types.
       done = []
       on.sort.each { |elt|
-        if elt.is_a?(Hash) and elt['name'] or elt['entity']# or elt['cloud_id']
-          with.sort.each { |other_elt|
-            # Figure out what convention this thing is using for resource identification
-            compare_a, compare_b = if elt['name'].nil? and elt["id"].nil? and !elt["entity"].nil? and !other_elt["entity"].nil?
-              [elt["entity"], other_elt["entity"]]
-            else
-              [elt, other_elt]
-            end
+        if elt.is_a?(Hash) and !MU::MommaCat.getChunkName(elt).first.nil?
+          elt_namestr, elt_location, elt_location_list = MU::MommaCat.getChunkName(elt)
 
-            if (compare_a['name'] and compare_b['name'] == compare_a['name']) or
-               (compare_a['name'].nil? and !compare_a["id"].nil? and compare_a["id"] == compare_b["id"])
-              break if elt == other_elt
+          with.sort.each { |other_elt|
+            other_elt_namestr, other_elt_location, other_elt_location_list = MU::MommaCat.getChunkName(other_elt)
+
+            # Case 1: The array element exists in both version of this array
+            if elt_namestr and other_elt_namestr and
+               elt_namestr == other_elt_namestr and
+               (elt_location.nil? or other_elt_location.nil? or
+                elt_location == other_elt_location or
+                !(elt_location_list & other_elt_location_list).empty?
+               )
               done << elt
               done << other_elt
-              namestr = if elt['type']
-                "#{elt['type']}[#{elt['name']}]"
-              elsif elt['name']
-                elt['name']
-              elsif elt['entity'] and elt["entity"]["id"]
-                elt['entity']['id']
+              break if elt == other_elt # if they're identical, we're done
+              report_data = diff(other_elt, elt, level: level+1, parents: parents + [elt_namestr], habitat: (elt_location || habitat))
+              if report_data and !report_data.empty?
+                report ||= {}
+                report[elt_namestr] = report_data
               end
-
-              diff(other_elt, elt, level: level+1, parents: parents + [namestr])
               break
             end
           }
@@ -152,43 +152,34 @@ class Hash
       }
       on_unique = (on - with) - done
       with_unique = (with - on) - done
-#    if on_unique.size > 0 or with_unique.size > 0
-#      if before_a != after_a
-#        MU.log "A BEFORE", MU::NOTICE, details: before_a
-#        MU.log "A AFTER", MU::NOTICE, details: after_a
-#      end
-#      if before_b != after_b
-#        MU.log "B BEFORE", MU::NOTICE, details: before_b
-#        MU.log "B AFTER", MU::NOTICE, details: after_b
-#      end
-#    end
+
+      # Case 2: This array entry exists in the old version, but not the new one
       on_unique.each { |e|
-        changes << if e.is_a?(Hash)
-          "- ".red+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
-        else
-          "- ".red+e.to_s
-        end
+        namestr, loc = MU::MommaCat.getChunkName(e)
+
+        report ||= {}
+        report[namestr] = { :action => :removed, :parents => parents, :value => e.clone }
+        report[namestr][:habitat] = loc if loc
       }
+
+      # Case 3: This array entry exists in the new version, but not the old one
       with_unique.each { |e|
-        changes << if e.is_a?(Hash)
-          "+ ".green+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n  "+(indent))
-        else
-          "+ ".green+e.to_s
-        end
+        namestr, loc = MU::MommaCat.getChunkName(e)
+
+        report ||= {}
+        report[namestr] = { :action => :added, :parents => parents, :value => e.clone }
+        report[namestr][:habitat] = loc if loc
       }
+
+    # A plain old leaf node of data
     else
       if on != with
-        changes << "-".red+" #{on.to_s}"
-        changes << "+".green+" #{with.to_s}"
+        report = { :action => :changed, :parents => parents, :oldvalue => on, :value => with.clone }
+        report[:habitat] = habitat if habitat
       end
     end
 
-    if changes.size > 0
-      puts tree
-      changes.each { |c|
-        puts indent+c
-      }
-    end
+    report.freeze
   end
 
   # Implement a merge! that just updates each hash leaf as needed, not 
@@ -212,8 +203,29 @@ class Hash
 end
 
 ENV['HOME'] = Etc.getpwuid(Process.uid).dir
+module MU
+
+  # For log entries that should only be logged when we're in verbose mode
+  DEBUG = 0.freeze
+  # For ordinary log entries
+  INFO = 1.freeze
+  # For more interesting log entries which are not errors
+  NOTICE = 2.freeze
+  # Log entries for non-fatal errors
+  WARN = 3.freeze
+  # Log entries for non-fatal errors
+  WARNING = 3.freeze
+  # Log entries for fatal errors
+  ERR = 4.freeze
+  # Log entries for fatal errors
+  ERROR = 4.freeze
+  # Log entries that will be held and displayed/emailed at the end of deploy,
+  # cleanup, etc.
+  SUMMARY = 5.freeze
+end
 
 require 'mu/logger'
+
 module MU
 
   # Subclass core thread so we can gracefully handle it when we hit system
@@ -273,8 +285,9 @@ module MU
   # Wrapper class for fatal Exceptions. Gives our internals something to
   # inherit that will log an error message appropriately before bubbling up.
   class MuError < StandardError
-    def initialize(message = nil, silent: false)
-      MU.log message, MU::ERR, details: caller[2] if !message.nil? and !silent
+    def initialize(message = nil, silent: false, details: nil)
+      details ||= caller[2]
+      MU.log message, MU::ERR, details: details if !message.nil? and !silent
       if MU.verbosity == MU::Logger::SILENT
         super ""
       else
@@ -620,25 +633,6 @@ module MU
     @@logger.log(msg, level, details: details, html: html, verbosity: verbosity, color: color)
   end
 
-  # For log entries that should only be logged when we're in verbose mode
-  DEBUG = 0.freeze
-  # For ordinary log entries
-  INFO = 1.freeze
-  # For more interesting log entries which are not errors
-  NOTICE = 2.freeze
-  # Log entries for non-fatal errors
-  WARN = 3.freeze
-  # Log entries for non-fatal errors
-  WARNING = 3.freeze
-  # Log entries for fatal errors
-  ERR = 4.freeze
-  # Log entries for fatal errors
-  ERROR = 4.freeze
-  # Log entries that will be held and displayed/emailed at the end of deploy,
-  # cleanup, etc.
-  SUMMARY = 5.freeze
-
-
   autoload :Cleanup, 'mu/cleanup'
   autoload :Deploy, 'mu/deploy'
   autoload :MommaCat, 'mu/mommacat'
@@ -652,7 +646,7 @@ module MU
     new_cfg = $MU_CFG.dup
     examples = {}
     MU::Cloud.supportedClouds.each { |cloud|
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+      cloudclass = MU::Cloud.cloudClass(cloud)
       begin
         if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
           cfg_blob = cloudclass.hosted_config
@@ -808,11 +802,7 @@ module MU
   # @param groomer [String]: The grooming agent to load.
   # @return [Class]: The class object implementing this groomer agent
   def self.loadGroomer(groomer)
-    if !File.size?(MU.myRoot+"/modules/mu/groomers/#{groomer.downcase}.rb")
-      raise MuError, "Requested to use unsupported grooming agent #{groomer}"
-    end
-    require "mu/groomers/#{groomer.downcase}"
-    return Object.const_get("MU").const_get("Groomer").const_get(groomer)
+    MU::Groomer.loadGroomer(groomer)
   end
 
   @@myRegion_var = nil
@@ -966,8 +956,7 @@ module MU
 
   @@myCloudDescriptor = nil
   if MU.myCloud
-    svrclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud).const_get("Server")
-    found = svrclass.find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
+    found = MU::Cloud.resourceClass(MU.myCloud, "Server").find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
 #    found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
     if !found.nil? and found.size == 1
       @@myCloudDescriptor = found.values.first
@@ -980,8 +969,7 @@ module MU
   def self.myVPCObj
     return nil if MU.myCloud.nil?
     return @@myVPCObj_var if @@myVPCObj_var
-    cloudclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud)
-    @@myVPCObj_var ||= cloudclass.myVPCObj
+    @@myVPCObj_var ||= MU::Cloud.cloudClass(MU.myCloud).myVPCObj
     @@myVPCObj_var
   end
 
@@ -1106,10 +1094,9 @@ module MU
 
     clouds = platform.nil? ? MU::Cloud.supportedClouds : [platform]
     clouds.each { |cloud|
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-      bucketname = cloudclass.adminBucketName(credentials)
+      bucketname = MU::Cloud.cloudClass(cloud).adminBucketName(credentials)
       begin
-        if platform or (cloudclass.hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
+        if platform or (MU::Cloud.cloudClass(cloud).hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
           return bucketname
         end
       end