cloud-mu 3.1.6 → 3.4.0

data/modules/mu/config/nosqldb.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/nosqldb.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/nosqldb.rb
     class NoSQLDB
 
       # Base configuration schema for a Bucket

data/modules/mu/config/notifier.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/notifier.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/notifier.rb
     class Notifier
 
       # Base configuration schema for a Notifier
@@ -36,12 +36,12 @@ module MU
               "items" => {
                 "type" => "object",
                 "description" => "A list of people or resources which should receive notifications",
-                "required" => ["endpoint"],
                 "properties" => {
                   "endpoint" => {
                     "type" => "string",
-                    "description" => "The endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number."
-                  }
+                    "description" => "Shorthand for an endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number. For complex cases, such as referencing an AWS Lambda function defined elsewhere in your Mu stack, use +resource+ instead."
+                  },
+                  "resource" => MU::Config::Ref.schema(desc: "A cloud resource that is a valid notification target for this notifier. For simple use cases, such as external email addresses or SMS, use +endpoint+ instead.")
                 }
               }
             }
@@ -56,23 +56,12 @@ module MU
       def self.validate(notifier, _configurator)
         ok = true
 
+
         if notifier['subscriptions']
           notifier['subscriptions'].each { |sub|
-            if !sub["type"]
-              if sub["endpoint"].match(/^http:/i)
-                sub["type"] = "http"
-              elsif sub["endpoint"].match(/^https:/i)
-                sub["type"] = "https"
-              elsif sub["endpoint"].match(/^sqs:/i)
-                sub["type"] = "sqs"
-              elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
-                sub["type"] = "sms"
-              elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
-                sub["type"] = "email"
-              else
-                MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR
-                ok = false
-              end
+            if !sub['endpoint'] and !sub['resource']
+              MU.log "Notifier '#{notifier['name']}' must specify either resource or endpoint in subscription", MU::ERR, details: sub
+              ok = false
             end
           }
         end

data/modules/mu/config/ref.rb

@@ -121,6 +121,10 @@ module MU
           @deploy_id = @mommacat.deploy_id
         end
 
+        # canonicalize the 'type' argument
+        _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        @type = cfg_plural if cfg_plural
+
         kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this
       end
 
@@ -130,10 +134,34 @@ module MU
         self.to_s <=> other.to_s
       end
 
+      # Lets callers access us like a {Hash}
+      # @param attribute [String,Symbol]
+      def [](attribute)
+        if respond_to?(attribute.to_sym)
+          send(attribute.to_sym)
+        else
+          nil
+        end
+      end
+
+      # Lets callers set attributes like a {Hash}
+      # @param attribute [String,Symbol]
+      def []=(attribute, value)
+        instance_variable_set("@#{attribute.to_s}".to_sym, value)
+        self.class.define_reader(attribute)
+      end
+
+      # Unset an attribute. Sort of. We can't actually do that, so nil it out
+      # and we get the behavior we want.
+      def delete(attribute)
+        attribute = ("@"+attribute).to_sym if attribute.to_s !~ /^@/
+        instance_variable_set(attribute.to_sym, nil)
+      end
+
       # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}.
       # @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+).
       # @return [Hash]
-      def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [])
+      def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [], any_type: false)
         parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1')
         desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource"
         schema = {
@@ -188,7 +216,9 @@ module MU
           }
         end
 
-        if !type.nil?
+        if any_type
+          schema["properties"]["type"].delete("enum")
+        elsif !type.nil?
           schema["required"] = ["type"]
           schema["properties"]["type"]["default"] = type
           schema["properties"]["type"]["enum"] = [type]
@@ -208,6 +238,13 @@ module MU
         schema
       end
 
+      # Is our +@type+ attribute a Mu-supported type, or some rando string?
+      # @return [Boolean]
+      def is_mu_type?
+        _shortclass, _cfg_name, type, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        !type.nil?
+      end
+
       # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment,
       # of the sort that would have been used to declare this reference in the
       # first place.
@@ -249,8 +286,22 @@ module MU
       # called in a live deploy, which is to say that if called during initial
       # configuration parsing, results may be incorrect.
       # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches.
-      def kitten(mommacat = @mommacat, shallow: false)
-        return nil if !@cloud or !@type
+      def kitten(mommacat = @mommacat, shallow: false, debug: false, cloud: nil)
+        cloud ||= @cloud
+        return nil if !cloud or !@type
+
+        _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        if cfg_plural
+          @type = cfg_plural # make sure this is the thing we expect
+        else
+          return nil # we don't do non-muish resources
+        end
+
+        loglevel = debug ? MU::NOTICE : MU::DEBUG
+
+        if debug
+          MU.log "this mf kitten", MU::WARN, details: caller
+        end
 
         if @obj
           @deploy_id ||= @obj.deploy_id
@@ -259,8 +310,16 @@ module MU
           return @obj
         end
 
-        if mommacat
-          @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: false)
+        if mommacat and caller.grep(/`findLitterMate'/).empty? # XXX the dumbest
+          MU.log "Looking for #{@type} #{@name} #{@id} in deploy #{mommacat.deploy_id}", loglevel
+          begin
+            @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: debug)
+          rescue StandardError => e
+            if e.message =~ /deadlock/
+              MU.log "Saw a recursive deadlock trying to fetch kitten for Ref object in deploy #{mmommacat.deploy_id}", MU::ERR, details: to_h
+            end
+            raise e
+          end
           if @obj # initialize missing attributes, if we can
             @id ||= @obj.cloud_id
             @mommacat ||= mommacat
@@ -283,7 +342,7 @@ end
           end
         end
 
-        if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id)) and !shallow
+        if !@obj and !(cloud == "Google" and @id and @type == "users" and MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(@id)) and !shallow
           try_deploy_id = @deploy_id
 
           begin
@@ -297,8 +356,20 @@ end
               [@habitat.to_s]
             end
 
+            MU.log "Ref#kitten calling findStray", loglevel, details: {
+              cloud: cloud,
+              type: @type,
+              name: @name,
+              cloud_id: @id,
+              deploy_id: try_deploy_id,
+              region: @region,
+              habitats: hab_arg,
+              credentials: @credentials,
+              dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
+            }
+
             found = MU::MommaCat.findStray(
-              @cloud,
+              cloud,
               @type,
               name: @name,
               cloud_id: @id,
@@ -308,10 +379,11 @@ end
               credentials: @credentials,
               dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
             )
+            MU.log "Ref#kitten results from findStray", loglevel, details: found
             @obj ||= found.first if found
           rescue MU::MommaCat::MultipleMatches => e
             if try_deploy_id.nil? and MU.deploy_id
-              MU.log "Attempting to narrow down #{@cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE
+              MU.log "Attempting to narrow down #{cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE
               try_deploy_id = MU.deploy_id
               retry
             else

data/modules/mu/config/role.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/role.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/role.rb
     class Role
 
       # Base configuration schema for a Group

data/modules/mu/config/schema_helpers.rb

@@ -22,10 +22,9 @@ module MU
     def self.defaultCloud
       configured = {}
       MU::Cloud.supportedClouds.each { |cloud|
-        cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
         if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty?
           configured[cloud] = $MU_CFG[cloud.downcase].size
-          configured[cloud] += 0.5 if cloudclass.hosted? # tiebreaker
+          configured[cloud] += 0.5 if MU::Cloud.cloudClass(cloud).hosted? # tiebreaker
         end
       }
       if configured.size > 0
@@ -34,8 +33,7 @@ module MU
         }.first
       else
         MU::Cloud.supportedClouds.each { |cloud|
-          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-          return cloud if cloudclass.hosted?
+          return cloud if MU::Cloud.cloudClass(cloud).hosted?
         }
         return MU::Cloud.supportedClouds.first
       end
@@ -83,9 +81,8 @@ module MU
     @@loadfails = []
     MU::Cloud.availableClouds.each { |cloud|
       next if @@loadfails.include?(cloud)
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
       begin
-        regions = cloudclass.listRegions()
+        regions = MU::Cloud.cloudClass(cloud).listRegions()
         @@allregions.concat(regions) if regions
       rescue MU::MuError => e
         @@loadfails << cloud
@@ -100,7 +97,7 @@ module MU
         @@allregions = []
         MU::Cloud.availableClouds.each { |cloud|
           next if @@loadfails.include?(cloud)
-          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+          cloudclass = MU::Cloud.cloudClass(cloud)
           begin
             return @@allregions if !cloudclass.listRegions()
             @@allregions.concat(cloudclass.listRegions())
@@ -178,27 +175,27 @@ module MU
       {
         "type" => "array",
         "items" => {
-            "type" => "object",
-            "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.",
-            "required" => ["name", "type"],
-            "additionalProperties" => false,
-            "properties" => {
-                "name" => {"type" => "string"},
-                "type" => {
-                    "type" => "string",
-                    "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] }
-                },
-                "phase" => {
-                  "type" => "string",
-                  "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.",
-                  "enum" => ["create", "groom"]
-                },
-                "no_create_wait" => {
-                    "type" => "boolean",
-                    "default" => false,
-                    "description" => "By default, it's assumed that we want to wait on our parents' creation phase, in addition to whatever is declared in this stanza. Setting this flag will bypass waiting on our parent resource's creation, so that our create or groom phase can instead depend only on the parent's groom phase. "
-                }
+          "type" => "object",
+          "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.",
+          "required" => ["name", "type"],
+          "additionalProperties" => false,
+          "properties" => {
+            "name" => {"type" => "string"},
+            "type" => {
+              "type" => "string",
+              "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] }
+            },
+            "phase" => {
+              "type" => "string",
+              "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.",
+              "enum" => ["create", "groom"]
+            },
+            "no_create_wait" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "By default, it's assumed that we want to wait on our parents' creation phase, in addition to whatever is declared in this stanza. Setting this flag will bypass waiting on our parent resource's creation, so that our create or groom phase can instead depend only on the parent's groom phase. "
             }
+          }
         }
       }
     end
@@ -244,7 +241,7 @@ module MU
       schema["title"] = type.to_s
 
       if cloud
-        cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(shortclass)
+        cloudclass = MU::Cloud.resourceClass(cloud, type)
 
         if cloudclass.respond_to?(:schema)
           _reqd, cloudschema = cloudclass.schema
@@ -298,11 +295,10 @@ module MU
           # schema information so that we set those defaults correctly.
           realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])
 
-            cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type)
-            _toplevel_required, cloudschema = cloudclass.schema(self)
+            _toplevel_required, cloudschema = MU::Cloud.resourceClass(item["cloud"], type).schema(self)
 
             newschema = schema_chunk["items"].dup
-            newschema["properties"].merge!(cloudschema)
+            MU::Config.schemaMerge(newschema["properties"], cloudschema, item["cloud"])
             newschema
           else
             schema_chunk["items"].dup
@@ -339,9 +335,9 @@ module MU
         return
       end
 
-      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud'])
-      shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type)
-      resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass)
+      cloudclass = MU::Cloud.cloudClass(kitten['cloud'])
+
+      resclass = MU::Cloud.resourceClass(kitten['cloud'], type)
 
       schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"]
       if !resclass.isGlobal?

data/modules/mu/config/search_domain.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/search_domain.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/search_domain.rb
     class SearchDomain
 
       # Base configuration schema for a SearchDomain

data/modules/mu/config/server.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/server.rb
     class Server
 
       # Verify that a server or server_pool has a valid LDAP config referencing
@@ -546,7 +546,7 @@ module MU
           "additionalProperties" => false,
           "description" => "Create individual server instances.",
           "properties" => {
-              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true),
+              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true, embedded_type: "server"),
               "bastion" => {
                 "type" => "boolean",
                 "default" => false,
@@ -650,20 +650,12 @@ module MU
           end
 
           if !server["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(server["vpc"]["subnet_name"]) and !configurator.nat_routes[server["vpc"]["subnet_name"]].empty?
-            server["dependencies"] << {
-              "type" => "server",
-              "name" => configurator.nat_routes[server["vpc"]["subnet_name"]],
-              "phase" => "groom"
-            }
+            MU::Config.addDependency(server, configurator.nat_routes[server["vpc"]["subnet_name"]], "server", phase: "groom", no_create_wait: true)
           elsif !server["vpc"]["name"].nil?
             siblingvpc = configurator.haveLitterMate?(server["vpc"]["name"], "vpcs")
             if siblingvpc and siblingvpc['bastion'] and
-               server['name'] != siblingvpc['bastion'].to_h['name']
-              server["dependencies"] << {
-                "type" => "server",
-                "name" => siblingvpc['bastion'].to_h['name'],
-                "phase" => "groom"
-              }
+               server['name'] != siblingvpc['bastion']['name']
+              MU::Config.addDependency(server, siblingvpc['bastion']['name'], "server", phase: "groom", no_create_wait: true)
             end
           end
         end

data/modules/mu/config/server_pool.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server_pool.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/server_pool.rb
     class ServerPool
 
       # Base configuration schema for a ServerPool
@@ -186,11 +186,7 @@ module MU
 
         if !pool["vpc"].nil?
           if !pool["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(pool["vpc"]["subnet_name"])
-            pool["dependencies"] << {
-              "type" => "pool",
-              "name" => configurator.nat_routes[pool["vpc"]["subnet_name"]],
-              "phase" => "groom"
-            }
+            MU::Config.addDependency(pool, configurator.nat_routes[pool["vpc"]["subnet_name"]], "server", phase: "groom", no_create_wait: true)
           end
         end
 # TODO make sure this is handled... somewhere
@@ -203,7 +199,7 @@ module MU
 #          }
 #        end
         if pool["basis"] and pool["basis"]["server"]
-          pool["dependencies"] << {"type" => "server", "name" => pool["basis"]["server"]}
+          MU::Config.addDependency(pool, pool["basis"]["server"], "server", phase: "groom")
         end
         if !pool['static_ip'].nil? and !pool['ip'].nil?
           ok = false

data/modules/mu/config/storage_pool.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/storage_pool.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/storage_pool.rb
     class StoragePool
 
       # Base configuration schema for a StoragePool

data/modules/mu/config/tail.rb

@@ -117,6 +117,16 @@ module MU
       def gsub(*args)
         to_s.gsub(*args)
       end
+
+      # Lets callers access us like a {Hash}
+      # @param attribute [String,Symbol]
+      def [](attribute)
+        if respond_to?(attribute.to_sym)
+          send(attribute.to_sym)
+        else
+          nil
+        end
+      end
     end
 
     # Wrapper method for creating a {MU::Config::Tail} object as a reference to