cloud-mu 3.2.0 → 3.5.0

data/modules/mu/adoption.rb

@@ -30,8 +30,7 @@ module MU
       :omnibus => "Jam everything into one monolothic configuration"
     }
 
-
-    def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false)
+    def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false, pattern: nil)
       @scraped = {}
       @clouds = clouds
       @types = types
@@ -49,6 +48,7 @@ module MU
       @habitats ||= []
       @scrub_mu_isms = scrub_mu_isms
       @merge = merge
+      @pattern = pattern
     end
 
     # Walk cloud providers with available credentials to discover resources
@@ -127,6 +127,7 @@ module MU
                 if obj.habitat and !cloudclass.listHabitats(credset).include?(obj.habitat)
                   next
                 end
+
                 # XXX apply any filters (e.g. MU-ID tags)
                 if obj.cloud_id.nil?
                   MU.log "This damn thing gave me no cloud id, what do I even do with that", MU::ERR, details: obj
@@ -292,7 +293,7 @@ module MU
                 start = Time.now
 
                 kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats, types: @types)
-                if kitten_cfg
+                if kitten_cfg and (!@pattern or @pattern.match(kitten_cfg['name']))
                   print "."
                   kitten_cfg.delete("credentials") if @target_creds
                   class_semaphore.synchronize {
@@ -791,8 +792,7 @@ module MU
         elsif hashcfg["id"] and !hashcfg["name"]
           hashcfg.delete("deploy_id")
         else
-          pp parent.cloud_desc
-          raise Incomplete, "Failed to resolve reference on behalf of #{parent}"
+          raise Incomplete.new "Failed to resolve reference on behalf of #{parent}", details: hashcfg
         end
         hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id
 

data/modules/mu/cleanup.rb

@@ -32,7 +32,7 @@ module MU
 
     # Resource types, in the order in which we generally have to clean them up
     # to disentangle them from one another.
-    TYPES_IN_ORDER = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
+    TYPES_IN_ORDER = ["Collection", "CDN", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "Job", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
 
     # Purge all resources associated with a deployment.
     # @param deploy_id [String]: The identifier of the deployment to remove (typically seen in the MU-ID tag on a resource).
@@ -52,6 +52,7 @@ module MU
       @onlycloud = onlycloud
       @skipcloud = skipcloud
       @ignoremaster = ignoremaster
+      @deploy_id = deploy_id
 
       if @skipcloud and @onlycloud # you actually mean noop
         @onlycloud = @skipcloud = false
@@ -217,51 +218,69 @@ module MU
       cloudclass = MU::Cloud.cloudClass(cloud)
       habitatclass = MU::Cloud.resourceClass(cloud, "Habitat")
 
-      projects = []
-      if habitats
-        projects = habitats
-      else
+      if !habitats
+        habitats = []
         if $MU_CFG and $MU_CFG[cloud.downcase] and
            $MU_CFG[cloud.downcase][credset] and
            $MU_CFG[cloud.downcase][credset]["project"]
 # XXX GCP credential schema needs an array for projects
-          projects << $MU_CFG[cloud.downcase][credset]["project"]
+          habitats << $MU_CFG[cloud.downcase][credset]["project"]
         end
         begin
-          projects.concat(cloudclass.listHabitats(credset, use_cache: false))
+          habitats.concat(cloudclass.listHabitats(credset, use_cache: false))
         rescue NoMethodError
         end
       end
 
-      if projects == []
-        projects << "" # dummy
+      if habitats == []
+        habitats << "" # dummy
         MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{region}", MU::NOTICE
       end
-      projects.uniq!
+      habitats.uniq!
 
       # We do these in an order that unrolls dependent resources
       # sensibly, and we hit :Collection twice because AWS
       # CloudFormation sometimes fails internally.
-      projectthreads = []
-      projects.each { |project|
-        if habitats and !habitats.empty? and project != ""
-          next if !habitats.include?(project)
+      habitat_threads = []
+      habitats.each { |habitat|
+        if habitats and !habitats.empty? and habitat != ""
+          next if !habitats.include?(habitat)
         end
-        if @habitatsused and !@habitatsused.empty? and project != ""
-          next if !@habitatsused.include?(project)
+        if @habitatsused and !@habitatsused.empty? and habitat != ""
+          next if !@habitatsused.include?(habitat)
         end
-        next if !habitatclass.isLive?(project, credset)
+        next if !habitatclass.isLive?(habitat, credset)
 
-        projectthreads << Thread.new {
+        habitat_threads << Thread.new {
+          Thread.current.thread_variable_set("name", "#{cloud}/#{credset}/#{habitat}/#{region}")
           Thread.abort_on_exception = false
-          if !cleanHabitat(cloud, credset, region, project, global_vs_region_semaphore, global_done)
+          if !cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done)
             had_failures = true
           end
         } # TYPES_IN_ORDER.each { |t|
-      } # projects.each { |project|
-      projectthreads.each do |t|
-        t.join
-      end
+      } # habitats.each { |habitat|
+
+      last_checkin = Time.now
+      begin
+        deletia = []
+        habitat_threads.each { |t|
+          if !t.status
+            t.join
+            deletia << t
+          end
+        }
+        deletia.each { |t|
+          habitat_threads.delete(t)
+        }
+        if (Time.now - last_checkin) > 120
+          list = habitat_threads.map { |t|
+            t.thread_variable_get("name") + (t.thread_variable_get("type") ? "/"+t.thread_variable_get("type") : "")
+          }
+          MU.log "Waiting on #{habitat_threads.size.to_s} habitat#{habitat_threads.size > 1 ? "s" : ""} in region #{region}", MU::NOTICE, details: list
+          last_checkin = Time.now
+        end
+        sleep 10 if !habitat_threads.empty?
+      end while !habitat_threads.empty?
 
       had_failures
     end
@@ -311,7 +330,8 @@ module MU
           next
         end
       }
-      had_failures = true
+
+      had_failures
     end
     private_class_method :cleanHabitat
 
@@ -322,6 +342,7 @@ module MU
     # @param flags [Hash]:
     # @param region [String]:
     def self.call_cleanup(type, credset, provider, flags, region)
+      Thread.current.thread_variable_set("type", type)
       if @mommacat.nil? or @mommacat.numKittens(types: [type]) > 0
         if @mommacat
 
@@ -344,7 +365,8 @@ module MU
           region: region,
           cloud: provider,
           flags: flags,
-          credentials: credset
+          credentials: credset,
+          deploy_id: @deploy_id
         )
       else
         true

data/modules/mu/cloud.rb

@@ -148,6 +148,12 @@ module MU
     # Stub base class; real implementations generated at runtime
     class NoSQLDB;
     end
+    # Stub base class; real implementations generated at runtime
+    class Job;
+    end
+    # Stub base class; real implementations generated at runtime
+    class CDN;
+    end
 
     # Denotes a resource implementation which is missing significant
     # functionality or is largely untested.
@@ -436,7 +442,29 @@ module MU
         :cfg_plural => "nosqldbs",
         :interface => self.const_get("NoSQLDB"),
         :deps_wait_on_my_creation => true,
-        :waits_on_parent_completion => true,
+        :waits_on_parent_completion => false,
+        :class => @@generic_class_methods,
+        :instance => @@generic_instance_methods + [:groom]
+      },
+      :Job => {
+        :has_multiples => false,
+        :can_live_in_vpc => false,
+        :cfg_name => "job",
+        :cfg_plural => "jobs",
+        :interface => self.const_get("Job"),
+        :deps_wait_on_my_creation => true,
+        :waits_on_parent_completion => false,
+        :class => @@generic_class_methods,
+        :instance => @@generic_instance_methods + [:groom]
+      },
+      :CDN => {
+        :has_multiples => false,
+        :can_live_in_vpc => false,
+        :cfg_name => "cdn",
+        :cfg_plural => "cdns",
+        :interface => self.const_get("CDN"),
+        :deps_wait_on_my_creation => true,
+        :waits_on_parent_completion => false,
         :class => @@generic_class_methods,
         :instance => @@generic_instance_methods + [:groom]
       }

data/modules/mu/cloud/dnszone.rb

@@ -29,8 +29,6 @@ module MU
 
       # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all
       # requested records in background and returns immediately.
-      # @param cfg [Array]: An array of parsed {MU::Config::BasketofKittens::dnszones::records} objects.
-      # @param target [String]: Optional target for the records to be created. Overrides targets embedded in cfg records.
       def self.createRecordsFromConfig(*flags)
         cloudclass = MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone")
         if !flags.nil? and flags.size == 1

data/modules/mu/cloud/machine_images.rb

@@ -94,7 +94,7 @@ module MU
           else
             begin
               Timeout.timeout(2) do
-                response = open("#{base_url}/#{cloud}.yaml").read
+                response = URI.open("#{base_url}/#{cloud}.yaml").read
                 images ||= {}
                 images.deep_merge!(YAML.load(response))
                 break

data/modules/mu/cloud/providers.rb

@@ -64,7 +64,12 @@ module MU
     # code for each of its supported resource type classes.
     failed = []
     MU::Cloud.supportedClouds.each { |cloud|
-      require "mu/providers/#{cloud.downcase}"
+      begin
+        require "mu/providers/#{cloud.downcase}"
+      rescue LoadError, Gem::MissingSpecError => e
+        MU.log "Error loading #{cloud} library, calls into this provider will fail", MU::ERR, details: e.message
+        next
+      end
       cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
       @@generic_class_methods_toplevel.each { |method|
         if !cloudclass.respond_to?(method)

data/modules/mu/cloud/resource_base.rb

@@ -227,6 +227,10 @@ module MU
               }
             end
 
+            MU::MommaCat.listOptionalTags.each_pair { |k, v|
+              @tags[k] ||= v if v
+            }
+
             if @cloudparentclass.respond_to?(:resourceInitHook)
               @cloudparentclass.resourceInitHook(self, @deploy)
             end
@@ -265,6 +269,7 @@ module MU
                 attr_accessor :mu_windows_name # XXX might be ok as reader now
               end 
             end
+            @tags["Name"] ||= @mu_name if @mu_name
           end
 
         end
@@ -733,7 +738,7 @@ module MU
                 MU.log "Couldn't find existing resource #{ext_deploy["cloud_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
                 @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: found.mu_name, triggering_node: @mu_name)
               elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"]
-                MU.log "#{ext_deploy["mu_name"]} / #{ext_deploy["deploy_id"]}"
+                MU.log "#{self}: Importing metadata for #{ext_deploy["cloud_type"]} #{ext_deploy["mu_name"]} from #{ext_deploy["deploy_id"]}"
                 found = MU::MommaCat.findStray(
                   @config['cloud'],
                   ext_deploy["cloud_type"],
@@ -743,8 +748,11 @@ module MU
                   dummy_ok: false
                 ).first
   
-                MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
-                @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
+                if found.nil?
+                  MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR
+                else
+                  @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
+                end
               else
                 MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR
               end
@@ -893,17 +901,18 @@ module MU
             elsif method == :notify
               if retval.nil?
                 MU.log self.to_s+" didn't return any metadata from notify", MU::WARN, details: @cloudobj.cloud_desc
+              else
+                retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil?
+                retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
+                @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
               end
-              retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil?
-              retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
-              @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
             end
             @method_semaphore.synchronize {
               @method_locks.delete(method)
             }
 
             @deploydata = @cloudobj.deploydata
-            @config = @cloudobj.config
+            @config = MU::Config.manxify(@cloudobj.config)
             retval
           end
         } # end instance method list

data/modules/mu/cloud/ssh_sessions.rb

@@ -125,7 +125,7 @@ module MU
                 end
               end
             end
-          rescue RuntimeError => e
+          rescue RuntimeError, IOError => e
             raise MU::Cloud::BootstrapTempFail, "Got #{e.inspect} performing initial SSH connect tasks, will try again"
           end
 
@@ -187,6 +187,10 @@ module MU
             retry
 #            rescue SystemCallError, Timeout::Error, Errno::ECONNRESET, Errno::EHOSTUNREACH, Net::SSH::Proxy::ConnectError, SocketError, Net::SSH::Disconnect, Net::SSH::AuthenticationFailed, IOError, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, MU::Cloud::NetSSHFail => e
           rescue SystemExit, Timeout::Error, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::ConnectionTimeout, Net::SSH::Proxy::ConnectError, Net::SSH::Exception, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::EPIPE, SocketError, IOError => e
+            if !active?
+              raise MuError, "Server #{@mu_name} disappeared while I was attempting to log into it"
+            end
+
             begin
               session.close if !session.nil?
             rescue Net::SSH::Disconnect, IOError => e

data/modules/mu/cloud/wrappers.rb

@@ -103,14 +103,23 @@ module MU
                 next
               end
 
-              found = cloudclass.find(args)
-              if !found.nil?
-                if found.is_a?(Hash)
-                  allfound.merge!(found)
-                else
-                  raise MuError, "#{cloudclass}.find returned a non-Hash result"
-                end
+              credsets = if args[:credentials]
+                [args[:credentials]]
+              else
+                cloudbase.listCredentials
               end
+
+              credsets.each { |creds|
+                args[:credentials] = creds
+                found = cloudclass.find(args)
+                if !found.nil?
+                  if found.is_a?(Hash)
+                    allfound.merge!(found)
+                  else
+                    raise MuError, "#{cloudclass}.find returned a non-Hash result"
+                  end
+                end
+              }
             rescue MuCloudResourceNotImplemented
             end
           }
@@ -126,6 +135,10 @@ module MU
             clouds = [params[:cloud]]
             params.delete(:cloud)
           end
+          params[:deploy_id] ||= MU.deploy_id
+          if !params[:deploy_id] or params[:deploy_id].empty?
+            raise MuError, "Can't call cleanup methods without a deploy id"
+          end
 
           clouds.each { |cloud|
             begin

data/modules/mu/config.rb

@@ -437,27 +437,27 @@ module MU
     # @param type [String]
     # @param phase [String]
     # @param no_create_wait [Boolean]
-    def self.addDependency(resource, name, type, phase: nil, no_create_wait: false)
-      if ![nil, "create", "groom"].include?(phase)
-        raise MuError, "Invalid phase '#{phase}' while adding dependency #{type} #{name} to #{resource['name']}"
+    def self.addDependency(resource, name, type, their_phase: "create", my_phase: nil)
+      if ![nil, "create", "groom"].include?(their_phase)
+        raise MuError, "Invalid their_phase '#{their_phase}' while adding dependency #{type} #{name} to #{resource['name']}"
       end
       resource['dependencies'] ||= []
       _shortclass, cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type)
 
       resource['dependencies'].each { |dep|
         if dep['type'] == cfg_name and dep['name'].to_s == name.to_s
-          dep["no_create_wait"] = no_create_wait
-          dep["phase"] = phase if phase
+          dep["their_phase"] = their_phase if their_phase
+          dep["my_phase"] = my_phase if my_phase
           return
         end
       }
 
       newdep = {
         "type" => cfg_name,
-        "name"  => name.to_s,
-        "no_create_wait" => no_create_wait
+        "name"  => name.to_s
       }
-      newdep["phase"] = phase if phase
+      newdep["their_phase"] = their_phase if their_phase
+      newdep["my_phase"] = my_phase if my_phase
 
       resource['dependencies'] << newdep
 
@@ -746,7 +746,7 @@ module MU
           next if !acl_include["name"] and !acl_include["rule_name"]
           acl_include["name"] ||= acl_include["rule_name"]
           if haveLitterMate?(acl_include["name"], "firewall_rules")
-            MU::Config.addDependency(descriptor, acl_include["name"], "firewall_rule", no_create_wait: (cfg_name == "vpc"))
+            MU::Config.addDependency(descriptor, acl_include["name"], "firewall_rule", my_phase: ((cfg_name == "vpc") ? "groom" : "create"))
           elsif acl_include["name"]
             MU.log shortclass.to_s+" #{descriptor['name']} depends on FirewallRule #{acl_include["name"]}, but no such rule declared.", MU::ERR
             ok = false
@@ -892,6 +892,10 @@ module MU
           deleteme = []
 
           resource["dependencies"].each { |dependency|
+            dependency["their_phase"] ||= dependency["phase"]
+            dependency.delete("phase")
+            dependency["my_phase"] ||= dependency["no_create_wait"] ? "groom" : "create"
+            dependency.delete("no_create_wait")
             # make sure the thing we depend on really exists
             sibling = haveLitterMate?(dependency['name'], dependency['type'])
             if !sibling
@@ -929,10 +933,22 @@ module MU
               end
             end
 
+            if dependency['their_phase'] == "groom"
+              sibling['dependencies'].each { |sib_dep|
+                next if sib_dep['type'] != cfg_name or sib_dep['their_phase'] != "groom"
+                cousin = haveLitterMate?(sib_dep['name'], sib_dep['type'])
+                if cousin and cousin['name'] == resource['name']
+                  MU.log "Circular dependency between #{type} #{resource['name']} <=> #{dependency['type']} #{dependency['name']}", MU::ERR, details: [ resource['name'] => dependency, sibling['name'] => sib_dep ]
+                  ok = false
+                end
+              }
+            end
+
             # Check for a circular relationship that will lead to a deadlock
             # when creating resource. This only goes one layer deep, and does
             # not consider groom-phase deadlocks.
-            if dependency['phase'] == "groom" or dependency['no_create_wait'] or (
+            if dependency['their_phase'] == "groom" or
+               dependency['my_phase'] == "groom" or (
                  !MU::Cloud.resourceClass(sibling['cloud'], type).deps_wait_on_my_creation and
                  !MU::Cloud.resourceClass(resource['cloud'], type).waits_on_parent_completion
                )
@@ -941,7 +957,7 @@ module MU
 
             if sibling['dependencies']
               sibling['dependencies'].each { |sib_dep|
-                next if sib_dep['type'] != cfg_name or sib_dep['no_create_wait']
+                next if sib_dep['type'] != cfg_name or sib_dep['my_phase'] == "groom"
                 cousin = haveLitterMate?(sib_dep['name'], sib_dep['type'])
                 if cousin and cousin['name'] == resource['name']
                   MU.log "Circular dependency between #{type} #{resource['name']} <=> #{dependency['type']} #{dependency['name']}", MU::ERR, details: [ resource['name'] => dependency, sibling['name'] => sib_dep ]
@@ -1238,7 +1254,7 @@ module MU
                     "port" => db["port"],
                     "sgs" => [cfg_name+server['name']]
                   }
-                  MU::Config.addDependency(ruleset, cfg_name+server['name'], "firewall_rule", no_create_wait: true)
+                  MU::Config.addDependency(ruleset, cfg_name+server['name'], "firewall_rule", my_phase: "groom")
                 end
               }
             }