cloud-mu 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd9a15a0e94a578919c0cb22bf5c95ee86b5b4a03fcc0ea1c35da2c161e8cf31
-  data.tar.gz: 04cdc9d9a70de97fdaf12021166b707921ec981d7ab2dc92cd1ebce491a9e130
+  metadata.gz: 87070670dfd1950848c17f897f49f087ada7a3155e3a5ef8e24ffb2c62583e7f
+  data.tar.gz: 8cee858ebde553ee022e56db98e90c62a28eb4a0b0247f74bd48a940956fe1c5
 SHA512:
-  metadata.gz: adfef4f231a946a3929b7f8026a9e338658dadfef3482942a46e8fdb9b10c4ca7d68aa457d135815a3878882d88a103ec1ebacea4bcb1ae4c45b070d85ceee06
-  data.tar.gz: 3a7b04d7d2486b05e8d413b78029b610bb7ba6bd8a33a1c24cf7b8bdf63dabaf9e0a1baab8408a66f995d05df719261248cbf27ee5b230a350109053ddb0420a
+  metadata.gz: 5ed8ad1fd6834f0b251079eb7d8b3f1805b0f88aa47486943f4bb7ad75196161c6c41a8e53364f755d46fe7fa6e51b8fb83ab518747369d07cdb0cb360689866
+  data.tar.gz: 53acff0bb1ea78fada8a9d8e6b3084ccb02209b42a1b19f99a80d792706cb55f1a2d5e97b48ee1fa54cc8b2ee8f7947feecd7a1465f24e18f9cdb86a499de8fa

data/Dockerfile

@@ -8,7 +8,7 @@ RUN df -h
 
 RUN apt-get update
 
-RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl openssh-client
 
 RUN apt-get upgrade -y
 

data/bin/mu-adopt

@@ -45,6 +45,7 @@ $opt = Optimist::options do
   opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
   opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
   opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false
+  opt :pattern, "Only adopt resources whose resource name would match this pattern. Must be a valid regular expression. Alphabetical characters will be treated case-insensitively.", :required => false, :type => :string
 end
 
 ok = true
@@ -60,6 +61,16 @@ if $opt[:diff]
   $opt[:savedeploys] = false
 end
 
+pattern = nil
+if $opt[:pattern]
+  begin
+    pattern = Regexp.new($opt[:pattern], true)
+  rescue RegexpError => e
+    MU.log "Invalid --pattern option: #{e.message}", MU::ERR
+    exit 1
+  end
+end
+
 types = []
 $opt[:types].each { |t|
   t_name = t.gsub(/-/, "_")
@@ -98,7 +109,7 @@ if !ok
   exit 1
 end
 
-adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes])
+adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes], pattern: pattern)
 found = adoption.scrapeClouds
 if found.nil? or found.empty?
   MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }

data/bin/mu-load-config.rb

@@ -134,7 +134,7 @@ def loadMuConfig(default_cfg_overrides = nil)
     }
   end
 
-  global_cfg = { "config_files" => [] }
+  global_cfg = { "config_files" => [], "overridden_keys" => [] }
   if File.exist?(cfgPath)
     global_cfg = YAML.load(File.read(cfgPath))
     global_cfg["config_files"] = [cfgPath]
@@ -147,6 +147,7 @@ def loadMuConfig(default_cfg_overrides = nil)
     if localfile
       global_cfg.merge!(localfile)
       global_cfg["config_files"] << "#{home}/.mu.yaml"
+      global_cfg["overridden_keys"] = localfile.keys
     end
   end
   if !global_cfg.has_key?("installdir")

data/bin/mu-run-tests

@@ -34,6 +34,7 @@ Usage:
 #{$0} [-m <#>] [-f] [-v] [specific test BoK to run [...]]
   EOS
   opt :max_threads, "Environment to set on creation.", :require => false, :default => 3, :type => :integer
+  opt :max_retries, "Number of times to retry failed tests in --dryrun mode.", :require => false, :default => 2, :type => :integer
   opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false
   opt :verbose, "Show more information while running", :require => false, :default => false
 end
@@ -121,8 +122,19 @@ def execCommand(cmd, results_stash)
   }
 
   ok = true
-  output = %x{#{cmd} 2>&1}
-  ok = false if $?.exitstatus != 0
+  retries = 0
+  begin
+    output = %x{#{cmd} 2>&1}
+    if $?.exitstatus != 0
+      ok = false
+      retries += 1
+      if $opts[:verbose] and !$opts[:full] and retries <= $opts[:max_retries]
+        puts "#{cmd} RETRY #{retries.to_s}".light_red
+      end
+    else
+      ok = true
+    end
+  end while !ok and !$opts[:full] and retries <= $opts[:max_retries]
 
   results_stash["output"] += output
 

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.2.0'
-  s.date        = '2020-06-16'
+  s.version     = '3.3.0'
+  s.date        = '2020-09-22'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -57,7 +57,7 @@ EOF
   s.add_runtime_dependency 'rack', "~> 2.0"
   s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
   s.add_runtime_dependency 'rubocop', '~> 0.58'
-  s.add_runtime_dependency 'rubyzip', "~> 2.0"
+  s.add_runtime_dependency 'rubyzip', "~> 2.3"
   s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"
   s.add_runtime_dependency 'solve', '~> 4.0'

data/modules/mu.rb

@@ -299,8 +299,8 @@ module MU
   # Wrapper class for temporary Exceptions. Gives our internals something to
   # inherit that will log a notice message appropriately before bubbling up.
   class MuNonFatal < StandardError
-    def initialize(message = nil, silent: false)
-      MU.log message, MU::NOTICE if !message.nil? and !silent
+    def initialize(message = nil, silent: false, details: nil)
+      MU.log message, MU::NOTICE, details: details if !message.nil? and !silent
       if MU.verbosity == MU::Logger::SILENT
         super ""
       else

data/modules/mu/adoption.rb

@@ -30,8 +30,7 @@ module MU
       :omnibus => "Jam everything into one monolothic configuration"
     }
 
-
-    def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false)
+    def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false, pattern: nil)
       @scraped = {}
       @clouds = clouds
       @types = types
@@ -49,6 +48,7 @@ module MU
       @habitats ||= []
       @scrub_mu_isms = scrub_mu_isms
       @merge = merge
+      @pattern = pattern
     end
 
     # Walk cloud providers with available credentials to discover resources
@@ -127,6 +127,7 @@ module MU
                 if obj.habitat and !cloudclass.listHabitats(credset).include?(obj.habitat)
                   next
                 end
+
                 # XXX apply any filters (e.g. MU-ID tags)
                 if obj.cloud_id.nil?
                   MU.log "This damn thing gave me no cloud id, what do I even do with that", MU::ERR, details: obj
@@ -292,7 +293,7 @@ module MU
                 start = Time.now
 
                 kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats, types: @types)
-                if kitten_cfg
+                if kitten_cfg and (!@pattern or @pattern.match(kitten_cfg['name']))
                   print "."
                   kitten_cfg.delete("credentials") if @target_creds
                   class_semaphore.synchronize {
@@ -791,8 +792,7 @@ module MU
         elsif hashcfg["id"] and !hashcfg["name"]
           hashcfg.delete("deploy_id")
         else
-          pp parent.cloud_desc
-          raise Incomplete, "Failed to resolve reference on behalf of #{parent}"
+          raise Incomplete.new "Failed to resolve reference on behalf of #{parent}", details: hashcfg
         end
         hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id
 

data/modules/mu/cleanup.rb

@@ -32,7 +32,7 @@ module MU
 
     # Resource types, in the order in which we generally have to clean them up
     # to disentangle them from one another.
-    TYPES_IN_ORDER = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
+    TYPES_IN_ORDER = ["Collection", "CDN", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "Job", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
 
     # Purge all resources associated with a deployment.
     # @param deploy_id [String]: The identifier of the deployment to remove (typically seen in the MU-ID tag on a resource).
@@ -52,6 +52,7 @@ module MU
       @onlycloud = onlycloud
       @skipcloud = skipcloud
       @ignoremaster = ignoremaster
+      @deploy_id = deploy_id
 
       if @skipcloud and @onlycloud # you actually mean noop
         @onlycloud = @skipcloud = false
@@ -217,51 +218,69 @@ module MU
       cloudclass = MU::Cloud.cloudClass(cloud)
       habitatclass = MU::Cloud.resourceClass(cloud, "Habitat")
 
-      projects = []
-      if habitats
-        projects = habitats
-      else
+      if !habitats
+        habitats = []
         if $MU_CFG and $MU_CFG[cloud.downcase] and
            $MU_CFG[cloud.downcase][credset] and
            $MU_CFG[cloud.downcase][credset]["project"]
 # XXX GCP credential schema needs an array for projects
-          projects << $MU_CFG[cloud.downcase][credset]["project"]
+          habitats << $MU_CFG[cloud.downcase][credset]["project"]
         end
         begin
-          projects.concat(cloudclass.listHabitats(credset, use_cache: false))
+          habitats.concat(cloudclass.listHabitats(credset, use_cache: false))
         rescue NoMethodError
         end
       end
 
-      if projects == []
-        projects << "" # dummy
+      if habitats == []
+        habitats << "" # dummy
         MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{region}", MU::NOTICE
       end
-      projects.uniq!
+      habitats.uniq!
 
       # We do these in an order that unrolls dependent resources
       # sensibly, and we hit :Collection twice because AWS
       # CloudFormation sometimes fails internally.
-      projectthreads = []
-      projects.each { |project|
-        if habitats and !habitats.empty? and project != ""
-          next if !habitats.include?(project)
+      habitat_threads = []
+      habitats.each { |habitat|
+        if habitats and !habitats.empty? and habitat != ""
+          next if !habitats.include?(habitat)
         end
-        if @habitatsused and !@habitatsused.empty? and project != ""
-          next if !@habitatsused.include?(project)
+        if @habitatsused and !@habitatsused.empty? and habitat != ""
+          next if !@habitatsused.include?(habitat)
         end
-        next if !habitatclass.isLive?(project, credset)
+        next if !habitatclass.isLive?(habitat, credset)
 
-        projectthreads << Thread.new {
+        habitat_threads << Thread.new {
+          Thread.current.thread_variable_set("name", "#{cloud}/#{credset}/#{habitat}/#{region}")
           Thread.abort_on_exception = false
-          if !cleanHabitat(cloud, credset, region, project, global_vs_region_semaphore, global_done)
+          if !cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done)
             had_failures = true
           end
         } # TYPES_IN_ORDER.each { |t|
-      } # projects.each { |project|
-      projectthreads.each do |t|
-        t.join
-      end
+      } # habitats.each { |habitat|
+
+      last_checkin = Time.now
+      begin
+        deletia = []
+        habitat_threads.each { |t|
+          if !t.status
+            t.join
+            deletia << t
+          end
+        }
+        deletia.each { |t|
+          habitat_threads.delete(t)
+        }
+        if (Time.now - last_checkin) > 120
+          list = habitat_threads.map { |t|
+            t.thread_variable_get("name") + (t.thread_variable_get("type") ? "/"+t.thread_variable_get("type") : "")
+          }
+          MU.log "Waiting on #{habitat_threads.size.to_s} habitat#{habitat_threads.size > 1 ? "s" : ""} in region #{region}", MU::NOTICE, details: list
+          last_checkin = Time.now
+        end
+        sleep 10 if !habitat_threads.empty?
+      end while !habitat_threads.empty?
 
       had_failures
     end
@@ -311,7 +330,8 @@ module MU
           next
         end
       }
-      had_failures = true
+
+      had_failures
     end
     private_class_method :cleanHabitat
 
@@ -322,6 +342,7 @@ module MU
     # @param flags [Hash]:
     # @param region [String]:
     def self.call_cleanup(type, credset, provider, flags, region)
+      Thread.current.thread_variable_set("type", type)
       if @mommacat.nil? or @mommacat.numKittens(types: [type]) > 0
         if @mommacat
 
@@ -344,7 +365,8 @@ module MU
           region: region,
           cloud: provider,
           flags: flags,
-          credentials: credset
+          credentials: credset,
+          deploy_id: @deploy_id
         )
       else
         true

data/modules/mu/cloud.rb

@@ -148,6 +148,12 @@ module MU
     # Stub base class; real implementations generated at runtime
     class NoSQLDB;
     end
+    # Stub base class; real implementations generated at runtime
+    class Job;
+    end
+    # Stub base class; real implementations generated at runtime
+    class CDN;
+    end
 
     # Denotes a resource implementation which is missing significant
     # functionality or is largely untested.
@@ -436,7 +442,29 @@ module MU
         :cfg_plural => "nosqldbs",
         :interface => self.const_get("NoSQLDB"),
         :deps_wait_on_my_creation => true,
-        :waits_on_parent_completion => true,
+        :waits_on_parent_completion => false,
+        :class => @@generic_class_methods,
+        :instance => @@generic_instance_methods + [:groom]
+      },
+      :Job => {
+        :has_multiples => false,
+        :can_live_in_vpc => false,
+        :cfg_name => "job",
+        :cfg_plural => "jobs",
+        :interface => self.const_get("Job"),
+        :deps_wait_on_my_creation => true,
+        :waits_on_parent_completion => false,
+        :class => @@generic_class_methods,
+        :instance => @@generic_instance_methods + [:groom]
+      },
+      :CDN => {
+        :has_multiples => false,
+        :can_live_in_vpc => false,
+        :cfg_name => "cdn",
+        :cfg_plural => "cdns",
+        :interface => self.const_get("CDN"),
+        :deps_wait_on_my_creation => true,
+        :waits_on_parent_completion => false,
         :class => @@generic_class_methods,
         :instance => @@generic_instance_methods + [:groom]
       }

data/modules/mu/cloud/dnszone.rb

@@ -29,8 +29,6 @@ module MU
 
       # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all
       # requested records in background and returns immediately.
-      # @param cfg [Array]: An array of parsed {MU::Config::BasketofKittens::dnszones::records} objects.
-      # @param target [String]: Optional target for the records to be created. Overrides targets embedded in cfg records.
       def self.createRecordsFromConfig(*flags)
         cloudclass = MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone")
         if !flags.nil? and flags.size == 1

data/modules/mu/cloud/resource_base.rb

@@ -227,6 +227,10 @@ module MU
               }
             end
 
+            MU::MommaCat.listOptionalTags.each_pair { |k, v|
+              @tags[k] ||= v if v
+            }
+
             if @cloudparentclass.respond_to?(:resourceInitHook)
               @cloudparentclass.resourceInitHook(self, @deploy)
             end
@@ -265,6 +269,7 @@ module MU
                 attr_accessor :mu_windows_name # XXX might be ok as reader now
               end 
             end
+            @tags["Name"] ||= @mu_name if @mu_name
           end
 
         end
@@ -893,10 +898,11 @@ module MU
             elsif method == :notify
               if retval.nil?
                 MU.log self.to_s+" didn't return any metadata from notify", MU::WARN, details: @cloudobj.cloud_desc
+              else
+                retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil?
+                retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
+                @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
               end
-              retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil?
-              retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
-              @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
             end
             @method_semaphore.synchronize {
               @method_locks.delete(method)

data/modules/mu/cloud/wrappers.rb

@@ -126,6 +126,10 @@ module MU
             clouds = [params[:cloud]]
             params.delete(:cloud)
           end
+          params[:deploy_id] ||= MU.deploy_id
+          if !params[:deploy_id] or params[:deploy_id].empty?
+            raise MuError, "Can't call cleanup methods without a deploy id"
+          end
 
           clouds.each { |cloud|
             begin

data/modules/mu/config.rb

@@ -437,7 +437,7 @@ module MU
     # @param type [String]
     # @param phase [String]
     # @param no_create_wait [Boolean]
-    def self.addDependency(resource, name, type, phase: nil, no_create_wait: false)
+    def self.addDependency(resource, name, type, phase: "create", no_create_wait: false)
       if ![nil, "create", "groom"].include?(phase)
         raise MuError, "Invalid phase '#{phase}' while adding dependency #{type} #{name} to #{resource['name']}"
       end

data/modules/mu/config/bucket.rb

@@ -50,6 +50,24 @@ module MU
               "default" => "index.html",
               "description" => "If +web_enabled+, return this object when \"diretory\" (a path not ending in a key/object) is invoked."
             },
+            "upload" => {
+              "type" => "array",
+              "items" => {
+                "type" => "object",
+                "description" => "Upload objects to a bucket, where supported",
+                "required" => ["source", "destination"],
+                "properties" => {
+                  "source" => {
+                    "type" => "string",
+                    "description" => "A file or directory to upload. If a directory is specified, it will be recursively mirrored to the bucket +destination+."
+                  },
+                  "destination" => {
+                    "type" => "string",
+                    "description" => "Path to which +source+ file(s) will be uploaded in the bucket, relative to +/+"
+                  }
+                }
+              }
+            },
             "policies" => {
               "type" => "array",
               "items" => MU::Config::Role.policy_primitive(subobjects: true, grant_to: true, permissions_optional: true, targets_optional: true)
@@ -59,12 +77,23 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::buckets}, bare and unvalidated.
-      # @param _bucket [Hash]: The resource to process and validate
+      # @param bucket [Hash]: The resource to process and validate
       # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(_bucket, _configurator)
+      def self.validate(bucket, _configurator)
         ok = true
 
+        if bucket['upload']
+          bucket['upload'].each { |batch|
+            if !File.exists?(batch['source'])
+              MU.log "Bucket '#{bucket['name']}' specifies upload for file/directory that does not exist", MU::ERR, details: batch
+              ok = false
+              next
+            end
+            batch['source'] = File.realpath(File.expand_path(batch['source']))
+          }
+        end
+
         ok
       end