cloud-mu 3.1.5 → 3.3.2

data/modules/mu/config/msg_queue.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/msg_queue.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/msg_queue.rb
     class MsgQueue
 
       # Base configuration schema for a MsgQueue

data/modules/mu/config/nosqldb.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/nosqldb.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/nosqldb.rb
     class NoSQLDB
 
       # Base configuration schema for a Bucket

data/modules/mu/config/notifier.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/notifier.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/notifier.rb
     class Notifier
 
       # Base configuration schema for a Notifier
@@ -36,12 +36,12 @@ module MU
               "items" => {
                 "type" => "object",
                 "description" => "A list of people or resources which should receive notifications",
-                "required" => ["endpoint"],
                 "properties" => {
                   "endpoint" => {
                     "type" => "string",
-                    "description" => "The endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number."
-                  }
+                    "description" => "Shorthand for an endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number. For complex cases, such as referencing an AWS Lambda function defined elsewhere in your Mu stack, use +resource+ instead."
+                  },
+                  "resource" => MU::Config::Ref.schema(desc: "A cloud resource that is a valid notification target for this notifier. For simple use cases, such as external email addresses or SMS, use +endpoint+ instead.")
                 }
               }
             }
@@ -56,23 +56,12 @@ module MU
       def self.validate(notifier, _configurator)
         ok = true
 
+
         if notifier['subscriptions']
           notifier['subscriptions'].each { |sub|
-            if !sub["type"]
-              if sub["endpoint"].match(/^http:/i)
-                sub["type"] = "http"
-              elsif sub["endpoint"].match(/^https:/i)
-                sub["type"] = "https"
-              elsif sub["endpoint"].match(/^sqs:/i)
-                sub["type"] = "sqs"
-              elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
-                sub["type"] = "sms"
-              elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
-                sub["type"] = "email"
-              else
-                MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR
-                ok = false
-              end
+            if !sub['endpoint'] and !sub['resource']
+              MU.log "Notifier '#{notifier['name']}' must specify either resource or endpoint in subscription", MU::ERR, details: sub
+              ok = false
             end
           }
         end

data/modules/mu/config/ref.rb

@@ -121,6 +121,10 @@ module MU
           @deploy_id = @mommacat.deploy_id
         end
 
+        # canonicalize the 'type' argument
+        _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        @type = cfg_plural if cfg_plural
+
         kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this
       end
 
@@ -130,10 +134,34 @@ module MU
         self.to_s <=> other.to_s
       end
 
+      # Lets callers access us like a {Hash}
+      # @param attribute [String,Symbol]
+      def [](attribute)
+        if respond_to?(attribute.to_sym)
+          send(attribute.to_sym)
+        else
+          nil
+        end
+      end
+
+      # Lets callers set attributes like a {Hash}
+      # @param attribute [String,Symbol]
+      def []=(attribute, value)
+        instance_variable_set("@#{attribute.to_s}".to_sym, value)
+        self.class.define_reader(attribute)
+      end
+
+      # Unset an attribute. Sort of. We can't actually do that, so nil it out
+      # and we get the behavior we want.
+      def delete(attribute)
+        attribute = ("@"+attribute).to_sym if attribute.to_s !~ /^@/
+        instance_variable_set(attribute.to_sym, nil)
+      end
+
       # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}.
       # @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+).
       # @return [Hash]
-      def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [])
+      def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [], any_type: false)
         parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1')
         desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource"
         schema = {
@@ -188,7 +216,9 @@ module MU
           }
         end
 
-        if !type.nil?
+        if any_type
+          schema["properties"]["type"].delete("enum")
+        elsif !type.nil?
           schema["required"] = ["type"]
           schema["properties"]["type"]["default"] = type
           schema["properties"]["type"]["enum"] = [type]
@@ -208,6 +238,13 @@ module MU
         schema
       end
 
+      # Is our +@type+ attribute a Mu-supported type, or some rando string?
+      # @return [Boolean]
+      def is_mu_type?
+        _shortclass, _cfg_name, type, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        !type.nil?
+      end
+
       # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment,
       # of the sort that would have been used to declare this reference in the
       # first place.
@@ -249,23 +286,46 @@ module MU
       # called in a live deploy, which is to say that if called during initial
       # configuration parsing, results may be incorrect.
       # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches.
-      def kitten(mommacat = @mommacat, shallow: false)
-        return nil if !@cloud or !@type
+      def kitten(mommacat = @mommacat, shallow: false, debug: false, cloud: nil)
+        cloud ||= @cloud
+        return nil if !cloud or !@type
+
+        _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        if cfg_plural
+          @type = cfg_plural # make sure this is the thing we expect
+        else
+          return nil # we don't do non-muish resources
+        end
+
+        loglevel = debug ? MU::NOTICE : MU::DEBUG
+
+        if debug
+          MU.log "this mf kitten", MU::WARN, details: caller
+        end
 
         if @obj
           @deploy_id ||= @obj.deploy_id
           @id ||= @obj.cloud_id
-          @name ||= @obj.config['name']
+          @name ||= @obj.config['name'] if @obj.config
           return @obj
         end
 
-        if mommacat
-          @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: false)
+        if mommacat and !caller.grep(/`findLitterMate'/) # XXX the dumbest
+          MU.log "Looking for #{@type} #{@name} #{@id} in deploy #{mommacat.deploy_id}", loglevel
+          begin
+            @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: debug)
+          rescue StandardError => e
+            if e.message =~ /deadlock/
+              MU.log "Saw a recursive deadlock trying to fetch kitten for Ref object in deploy #{mmommacat.deploy_id}", MU::ERR, details: to_h
+            end
+            raise e
+          end
           if @obj # initialize missing attributes, if we can
             @id ||= @obj.cloud_id
             @mommacat ||= mommacat
             @obj.intoDeploy(@mommacat) # make real sure these are set
             @deploy_id ||= mommacat.deploy_id
+
             if !@name
               if @obj.config and @obj.config['name']
                 @name = @obj.config['name']
@@ -282,7 +342,8 @@ end
           end
         end
 
-        if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id)) and !shallow
+        if !@obj and !(cloud == "Google" and @id and @type == "users" and MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(@id)) and !shallow
+          try_deploy_id = @deploy_id
 
           begin
             hab_arg = if @habitat.nil?
@@ -295,27 +356,44 @@ end
               [@habitat.to_s]
             end
 
+            MU.log "Ref#kitten calling findStray", loglevel, details: {
+              cloud: cloud,
+              type: @type,
+              name: @name,
+              cloud_id: @id,
+              deploy_id: try_deploy_id,
+              region: @region,
+              habitats: hab_arg,
+              credentials: @credentials,
+              dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
+            }
+
             found = MU::MommaCat.findStray(
-              @cloud,
+              cloud,
               @type,
               name: @name,
               cloud_id: @id,
-              deploy_id: @deploy_id,
+              deploy_id: try_deploy_id,
               region: @region,
               habitats: hab_arg,
               credentials: @credentials,
               dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
             )
+            MU.log "Ref#kitten results from findStray", loglevel, details: found
             @obj ||= found.first if found
+          rescue MU::MommaCat::MultipleMatches => e
+            if try_deploy_id.nil? and MU.deploy_id
+              MU.log "Attempting to narrow down #{cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE
+              try_deploy_id = MU.deploy_id
+              retry
+            else
+              raise e
+            end
           rescue ThreadError => e
             # Sometimes MommaCat calls us in a potential deadlock situation;
             # don't be the cause of a fatal error if so, we don't need this
             # object that badly.
             raise e if !e.message.match(/recursive locking/)
-rescue SystemExit
-# XXX this is temporary, to cope with some debug stuff that's in findStray
-# for the nonce
-return
           end
         end
 

data/modules/mu/config/role.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/role.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/role.rb
     class Role
 
       # Base configuration schema for a Group

data/modules/mu/config/schema_helpers.rb

@@ -22,10 +22,9 @@ module MU
     def self.defaultCloud
       configured = {}
       MU::Cloud.supportedClouds.each { |cloud|
-        cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
         if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty?
           configured[cloud] = $MU_CFG[cloud.downcase].size
-          configured[cloud] += 0.5 if cloudclass.hosted? # tiebreaker
+          configured[cloud] += 0.5 if MU::Cloud.cloudClass(cloud).hosted? # tiebreaker
         end
       }
       if configured.size > 0
@@ -34,8 +33,7 @@ module MU
         }.first
       else
         MU::Cloud.supportedClouds.each { |cloud|
-          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-          return cloud if cloudclass.hosted?
+          return cloud if MU::Cloud.cloudClass(cloud).hosted?
         }
         return MU::Cloud.supportedClouds.first
       end
@@ -83,9 +81,8 @@ module MU
     @@loadfails = []
     MU::Cloud.availableClouds.each { |cloud|
       next if @@loadfails.include?(cloud)
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
       begin
-        regions = cloudclass.listRegions()
+        regions = MU::Cloud.cloudClass(cloud).listRegions()
         @@allregions.concat(regions) if regions
       rescue MU::MuError => e
         @@loadfails << cloud
@@ -100,7 +97,7 @@ module MU
         @@allregions = []
         MU::Cloud.availableClouds.each { |cloud|
           next if @@loadfails.include?(cloud)
-          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+          cloudclass = MU::Cloud.cloudClass(cloud)
           begin
             return @@allregions if !cloudclass.listRegions()
             @@allregions.concat(cloudclass.listRegions())
@@ -178,27 +175,27 @@ module MU
       {
         "type" => "array",
         "items" => {
-            "type" => "object",
-            "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.",
-            "required" => ["name", "type"],
-            "additionalProperties" => false,
-            "properties" => {
-                "name" => {"type" => "string"},
-                "type" => {
-                    "type" => "string",
-                    "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] }
-                },
-                "phase" => {
-                  "type" => "string",
-                  "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.",
-                  "enum" => ["create", "groom"]
-                },
-                "no_create_wait" => {
-                    "type" => "boolean",
-                    "default" => false,
-                    "description" => "By default, it's assumed that we want to wait on our parents' creation phase, in addition to whatever is declared in this stanza. Setting this flag will bypass waiting on our parent resource's creation, so that our create or groom phase can instead depend only on the parent's groom phase. "
-                }
+          "type" => "object",
+          "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.",
+          "required" => ["name", "type"],
+          "additionalProperties" => false,
+          "properties" => {
+            "name" => {"type" => "string"},
+            "type" => {
+              "type" => "string",
+              "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] }
+            },
+            "phase" => {
+              "type" => "string",
+              "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.",
+              "enum" => ["create", "groom"]
+            },
+            "no_create_wait" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "By default, it's assumed that we want to wait on our parents' creation phase, in addition to whatever is declared in this stanza. Setting this flag will bypass waiting on our parent resource's creation, so that our create or groom phase can instead depend only on the parent's groom phase. "
             }
+          }
         }
       }
     end
@@ -244,7 +241,7 @@ module MU
       schema["title"] = type.to_s
 
       if cloud
-        cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(shortclass)
+        cloudclass = MU::Cloud.resourceClass(cloud, type)
 
         if cloudclass.respond_to?(:schema)
           _reqd, cloudschema = cloudclass.schema
@@ -276,15 +273,20 @@ module MU
             schema_chunk["properties"]["creation_style"] != "existing"
           schema_chunk["properties"].each_pair { |key, subschema|
             shortclass = if conf_chunk[key]
-              shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key)
+              shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key, false)
               shortclass
             else
               nil
             end
 
             new_val = applySchemaDefaults(conf_chunk[key], subschema, depth+1, conf_chunk, type: shortclass).dup
-
-            conf_chunk[key] = Marshal.load(Marshal.dump(new_val)) if !new_val.nil?
+            if !new_val.nil?
+              begin
+                conf_chunk[key] = Marshal.load(Marshal.dump(new_val))
+              rescue TypeError
+                conf_chunk[key] = new_val.clone
+              end
+            end
           }
         end
       elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array)
@@ -293,11 +295,10 @@ module MU
           # schema information so that we set those defaults correctly.
           realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])
 
-            cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type)
-            _toplevel_required, cloudschema = cloudclass.schema(self)
+            _toplevel_required, cloudschema = MU::Cloud.resourceClass(item["cloud"], type).schema(self)
 
             newschema = schema_chunk["items"].dup
-            newschema["properties"].merge!(cloudschema)
+            MU::Config.schemaMerge(newschema["properties"], cloudschema, item["cloud"])
             newschema
           else
             schema_chunk["items"].dup
@@ -334,9 +335,9 @@ module MU
         return
       end
 
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud'])
-      shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type)
-      resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass)
+      cloudclass = MU::Cloud.cloudClass(kitten['cloud'])
+
+      resclass = MU::Cloud.resourceClass(kitten['cloud'], type)
 
       schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"]
       if !resclass.isGlobal?

data/modules/mu/config/search_domain.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/search_domain.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/search_domain.rb
     class SearchDomain
 
       # Base configuration schema for a SearchDomain

data/modules/mu/config/server.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/server.rb
     class Server
 
       # Verify that a server or server_pool has a valid LDAP config referencing
@@ -546,7 +546,7 @@ module MU
           "additionalProperties" => false,
           "description" => "Create individual server instances.",
           "properties" => {
-              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true),
+              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true, embedded_type: "server"),
               "bastion" => {
                 "type" => "boolean",
                 "default" => false,
@@ -625,6 +625,13 @@ module MU
         server['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
         ok = false if !MU::Config::Server.checkVaultRefs(server)
 
+        server['groomer'] ||= self.defaultGroomer
+        groomclass = MU::Groomer.loadGroomer(server['groomer'])
+        if !groomclass.available?(server['platform'].match(/^win/))
+          MU.log "Groomer #{server['groomer']} for #{server['name']} is missing or has incomplete dependencies", MU::ERR
+          ok = false
+        end
+
         if server["cloud"] != "Azure"
           server['dependencies'] << configurator.adminFirewallRuleset(vpc: server['vpc'], region: server['region'], cloud: server['cloud'], credentials: server['credentials'])
         end
@@ -643,20 +650,12 @@ module MU
           end
 
           if !server["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(server["vpc"]["subnet_name"]) and !configurator.nat_routes[server["vpc"]["subnet_name"]].empty?
-            server["dependencies"] << {
-              "type" => "server",
-              "name" => configurator.nat_routes[server["vpc"]["subnet_name"]],
-              "phase" => "groom"
-            }
+            MU::Config.addDependency(server, configurator.nat_routes[server["vpc"]["subnet_name"]], "server", phase: "groom", no_create_wait: true)
           elsif !server["vpc"]["name"].nil?
             siblingvpc = configurator.haveLitterMate?(server["vpc"]["name"], "vpcs")
             if siblingvpc and siblingvpc['bastion'] and
-               server['name'] != siblingvpc['bastion'].to_h['name']
-              server["dependencies"] << {
-                "type" => "server",
-                "name" => siblingvpc['bastion'].to_h['name'],
-                "phase" => "groom"
-              }
+               server['name'] != siblingvpc['bastion']['name']
+              MU::Config.addDependency(server, siblingvpc['bastion']['name'], "server", phase: "groom", no_create_wait: true)
             end
           end
         end