cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/modules/mu/config.rb

@@ -39,9 +39,10 @@ module MU
       rescue NoMethodError
         "AWS"
       end
-      if MU::Cloud::Google.hosted
+# XXX this can be more generic (loop through supportedClouds and try this)
+      if MU::Cloud::Google.hosted?
         "Google"
-      elsif MU::Cloud::AWS.hosted
+      elsif MU::Cloud::AWS.hosted?
         "AWS"
       end
     end
@@ -765,14 +766,27 @@ module MU
     # @param name [String]: The name of the resource being checked
     # @param type [String]: The type of resource being checked
     # @return [Boolean]
-    def haveLitterMate?(name, type)
+    def haveLitterMate?(name, type, has_multiple: false)
       @kittencfg_semaphore.synchronize {
+        matches = []
         shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
-        @kittens[cfg_plural].each { |kitten|
-          return kitten if kitten['name'] == name.to_s
-        }
+        if @kittens[cfg_plural]
+          @kittens[cfg_plural].each { |kitten|
+            if kitten['name'] == name.to_s or kitten['virtual_name'] == name.to_s
+              if has_multiple
+                matches << kitten
+              else
+                return kitten
+              end
+            end
+          }
+        end
+        if has_multiple
+          return matches
+        else
+          return false
+        end
       }
-      false
     end
 
     # Remove a resource from the current stack
@@ -782,13 +796,15 @@ module MU
       @kittencfg_semaphore.synchronize {
         shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
         deletia = nil
-        @kittens[cfg_plural].each { |kitten|
-          if kitten['name'] == name
-            deletia = kitten
-            break
-          end
-        }
-        @kittens[type].delete(deletia) if !deletia.nil?
+        if @kittens[cfg_plural]
+          @kittens[cfg_plural].each { |kitten|
+            if kitten['name'] == name
+              deletia = kitten
+              break
+            end
+          }
+          @kittens[type].delete(deletia) if !deletia.nil?
+        end
       }
     end
 
@@ -868,6 +884,9 @@ module MU
       # Does this resource go in a VPC?
       if !descriptor["vpc"].nil? and !delay_validation
         descriptor['vpc']['cloud'] = descriptor['cloud']
+        if descriptor['credentials']
+          descriptor['vpc']['credentials'] ||= descriptor['credentials']
+        end
         if descriptor['vpc']['region'].nil? and !descriptor['region'].nil? and !descriptor['region'].empty? and descriptor['vpc']['cloud'] != "Google"
           descriptor['vpc']['region'] = descriptor['region']
         end
@@ -896,6 +915,7 @@ module MU
                                   self,
                                   dflt_region: descriptor['region'],
                                   is_sibling: true,
+                                  credentials: descriptor['credentials'],
                                   sibling_vpcs: @kittens['vpcs'])
             ok = false
           end
@@ -907,11 +927,19 @@ module MU
           if !MU::Config::VPC.processReference(descriptor["vpc"], cfg_plural,
                                   "#{shortclass} #{descriptor['name']}",
                                   self,
+                                  credentials: descriptor['credentials'],
                                   dflt_region: descriptor['region'])
             MU.log "insertKitten was called from #{caller[0]}", MU::ERR
             ok = false
           end
         end
+
+        # if we didn't specify credentials but can inherit some from our target
+        # VPC, do so
+        if descriptor["vpc"]["credentials"]
+          descriptor["credentials"] ||= descriptor["vpc"]["credentials"] 
+        end
+
         # Clean crud out of auto-created VPC declarations so they don't trip
         # the schema validator when it's invoked later.
         if !["server", "server_pool", "database"].include?(cfg_name)
@@ -933,7 +961,12 @@ module MU
          ["server", "server_pool", "database"].include?(cfg_name))
         descriptor['ingress_rules'] ||= []
 
-        acl = {"name" => fwname, "rules" => descriptor['ingress_rules'], "region" => descriptor['region'] }
+        acl = {
+          "name" => fwname,
+          "rules" => descriptor['ingress_rules'],
+          "region" => descriptor['region'],
+          "credentials" => descriptor["credentials"]
+        }
         acl["vpc"] = descriptor['vpc'].dup if descriptor['vpc']
         ["optional_tags", "tags", "cloud", "project"].each { |param|
           acl[param] = descriptor[param] if descriptor[param]
@@ -992,6 +1025,7 @@ module MU
         descriptor["alarms"].each { |alarm|
           alarm["name"] = "#{cfg_name}-#{descriptor["name"]}-#{alarm["name"]}"
           alarm['dimensions'] = [] if !alarm['dimensions']
+          alarm["credentials"] = descriptor["credentials"]
           alarm["#TARGETCLASS"] = cfg_name
           alarm["#TARGETNAME"] = descriptor['name']
           alarm['cloud'] = descriptor['cloud']
@@ -1091,9 +1125,13 @@ module MU
         if ok
           parser = Object.const_get("MU").const_get("Cloud").const_get(descriptor["cloud"]).const_get(shortclass.to_s)
           plain_descriptor = MU::Config.manxify(Marshal.load(Marshal.dump(descriptor)))
-          return false if !parser.validateConfig(plain_descriptor, self)
+          passed = parser.validateConfig(plain_descriptor, self)
 
-          descriptor.merge!(plain_descriptor)
+          if passed
+            descriptor.merge!(plain_descriptor)
+          else
+            ok = false
+          end
           descriptor['#MU_VALIDATED'] = true
         end
 
@@ -1123,6 +1161,15 @@ module MU
       }
     end
 
+    # Configuration chunk for choosing a set of cloud credentials
+    # @return [Hash]
+    def self.credentials_primitive
+      {
+          "type" => "string",
+          "description" => "Specify a non-default set of credentials to use when authenticating to cloud provider APIs, as listed in `mu.yaml` under each provider's subsection. If "
+      }
+    end
+
     # Configuration chunk for creating resource tags as an array of key/value
     # pairs.
     # @return [Hash]
@@ -1178,7 +1225,7 @@ module MU
     # @param cloud [String]: The parent resource's cloud plugin identifier
     # @param region [String]: Cloud provider region, if applicable.
     # @return [Hash<String>]: A dependency description that the calling resource can then add to itself.
-    def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil)
+    def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil)
       if !cloud or (cloud == "AWS" and !region)
         raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider"
       end
@@ -1189,6 +1236,7 @@ module MU
       hosts << "#{admin_ip}/32" if admin_ip
       hosts.uniq!
       name = "admin"
+      name += credentials.to_s if credentials
       realvpc = nil
 
       if vpc
@@ -1223,9 +1271,9 @@ module MU
         ]
       end
 
-      acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true}
+      acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
       acl.delete("vpc") if !acl["vpc"]
-      acl["region"] == region if !region.nil? and !region.empty?
+      acl["region"] = region if !region.nil? and !region.empty?
       @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl)
       return {"type" => "firewall_rule", "name" => name}
     end
@@ -1438,12 +1486,15 @@ module MU
     # TODO check for loops
     def self.check_dependencies(config)
       ok = true
+
       config.each { |type|
         if type.instance_of?(Array)
           type.each { |container|
             if container.instance_of?(Array)
               container.each { |resource|
                 if resource.kind_of?(Hash) and resource["dependencies"] != nil
+                  append = []
+                  delete = []
                   resource["dependencies"].each { |dependency|
                     collection = dependency["type"]+"s"
                     found = false
@@ -1452,6 +1503,14 @@ module MU
                       config[collection].each { |service|
                         names_seen << service["name"].to_s
                         found = true if service["name"].to_s == dependency["name"].to_s
+                        if service["virtual_name"]
+                          names_seen << service["virtual_name"].to_s
+                          found = true if service["virtual_name"].to_s == dependency["name"].to_s
+                          append_me = dependency.dup
+                          append_me['name'] = service['name']
+                          append << append_me
+                          delete << dependency
+                        end
                       }
                     end
                     if !found
@@ -1459,6 +1518,15 @@ module MU
                       ok = false
                     end
                   }
+                  if append.size > 0
+                    append.uniq!
+                    resource["dependencies"].concat(append)
+                  end
+                  if delete.size > 0
+                    delete.each { |delete_me|
+                      resource["dependencies"].delete(delete_me)
+                    }
+                  end
                 end
               }
             end
@@ -1534,23 +1602,31 @@ module MU
     # @param type [String]: The type of resource this is ("servers" etc)
     def inheritDefaults(kitten, type)
       kitten['cloud'] ||= MU::Config.defaultCloud
+      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud'])
+      shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
+      resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass)
+
+      schema_fields = ["us_only", "scrub_mu_isms", "credentials"]
+      if !resclass.isGlobal?
+        schema_fields << "region"
+      end
 
-      schema_fields = ["region", "us_only", "scrub_mu_isms"]
       if kitten['cloud'] == "Google"
-        kitten["project"] ||= MU::Cloud::Google.defaultProject
+        kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials'])
         schema_fields << "project"
         if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and
+           !resclass.isGlobal? and
            ![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS'])
-          if !$MU_CFG['google'] or !$MU_CFG['google']['region']
-            raise ValidationError, "Google resource declared without a region, but no default Google region declared in mu.yaml"
+          if MU::Cloud::Google.myRegion((kitten['credentials'])).nil?
+            raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set" 
           end
-          kitten['region'] ||= $MU_CFG['google']['region']
+          kitten['region'] ||= MU::Cloud::Google.myRegion(kitten['credentials'])
         end
-      else
-        if !$MU_CFG['aws'] or !$MU_CFG['aws']['region']
-          raise ValidationError, "AWS resource declared without a region, but no default AWS region declared in mu.yaml"
+      elsif !resclass.isGlobal?
+        if MU::Cloud::AWS.myRegion.nil?
+          raise ValidationError, "AWS resource declared without a region, but no default AWS region found"
         end
-        kitten['region'] ||= $MU_CFG['aws']['region']
+        kitten['region'] ||= MU::Cloud::AWS.myRegion
       end
 
       kitten['us_only'] ||= @config['us_only']
@@ -1559,13 +1635,16 @@ module MU
       kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms']
       kitten['scrub_mu_isms'] ||= false
 
+      kitten['credentials'] ||= @config['credentials']
+      kitten['credentials'] ||= cloudclass.credConfig(name_only: true)
+
       kitten["dependencies"] ||= []
 
       # Make sure the schema knows about these "new" fields, so that validation
       # doesn't trip over them.
       schema_fields.each { |field|
         if @@schema["properties"][field]
-          MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG
+          MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG, details: @@schema["properties"][field]
           @@schema["properties"][type]["items"]["properties"][field] ||= @@schema["properties"][field]
         end
       }
@@ -1606,12 +1685,23 @@ module MU
 
       # Make sure validation has been called for all on-the-fly generated
       # resources.
-      types.each { |type|
-        @kittens[type].each { |descriptor|
-          if !descriptor["#MU_VALIDATED"]
-            ok = false if !insertKitten(descriptor, type)
-          end
+      validated_something_new = false
+      begin
+        validated_something_new = false
+        types.each { |type|
+          @kittens[type].each { |descriptor|
+            if !descriptor["#MU_VALIDATED"]
+              validated_something_new = true
+              ok = false if !insertKitten(descriptor, type)
+            end
+          }
         }
+      end while validated_something_new
+
+      # Do another pass of resolving intra-stack VPC peering, in case an
+      # early-parsing VPC needs more details from a later-parsing one
+      @kittens["vpcs"].each { |vpc|
+        ok = false if !MU::Config::VPC.resolvePeers(vpc, self)
       }
 
       # add some default holes to allow dependent instances into databases
@@ -1779,7 +1869,7 @@ module MU
         prefixes << "# **REQUIRED**" if required and schema['default'].nil?
         prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
         prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
-        if !schema['enum'].nil?
+        if !schema['enum'].nil? and !schema["enum"].empty?
           prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**"
         elsif !schema['pattern'].nil?
           # XXX unquoted regex chars confuse the hell out of YARD. How do we
@@ -1871,10 +1961,10 @@ module MU
         },
         "project" => {
           "type" => "string",
-          "description" => "GOOGLE: The project into which to deploy resources",
-          "default" => MU::Cloud::Google.defaultProject
+          "description" => "GOOGLE: The project into which to deploy resources"
         },
         "region" => MU::Config.region_primitive,
+        "credentials" => MU::Config.credentials_primitive,
         "us_only" => {
             "type" => "boolean",
             "description" => "For resources which span regions, restrict to regions inside the United States",
@@ -1983,8 +2073,13 @@ module MU
           "type" => "array",
           "items" => schemaclass.schema
         }
+        @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["virtual_name"] = {
+          "description" => "Internal use.",
+          "type" => "string"
+        }
         @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["dependencies"] = MU::Config.dependencies_primitive
         @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["cloud"] = MU::Config.cloud_primitive
+        @@schema["properties"][cfg[:cfg_plural]]["items"]["properties"]["credentials"] = MU::Config.credentials_primitive
         @@schema["properties"][cfg[:cfg_plural]]["items"]["title"] = type.to_s
       rescue NameError => e
         failed << type

data/modules/mu/config/alarm.rb

@@ -263,6 +263,31 @@ module MU
           }
         end
 
+        if alarm["enable_notifications"]
+          if !alarm["notification_group"].match(/^arn:/i)
+            if !configurator.haveLitterMate?(alarm["notification_group"], "notifiers")
+              notifier = {
+                "name" => alarm["notification_group"],
+                "region" => alarm["region"],
+                "cloud" => alarm["cloud"],
+                "credentials" => alarm["credentials"],
+                "subscriptions" => [
+                  {
+                    "endpoint" => alarm["notification_endpoint"],
+                    "type" => alarm["notification_type"],
+                  }
+                ]
+              }
+              ok = false if !configurator.insertKitten(notifier, "notifiers")
+            end
+            alarm["dependencies"] ||= []
+            alarm["dependencies"] << {
+              "name" => alarm["notification_group"],
+              "type" => "notifier"
+            }
+          end
+        end
+
         ok
       end
 

data/modules/mu/config/cache_cluster.rb

@@ -148,10 +148,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::cache_clusters}, bare and unvalidated.
-      # @param cache [Hash]: The resource to process and validate
+      # @param cluster [Hash]: The resource to process and validate
       # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(cache, configurator)
+      def self.validate(cluster, configurator)
         ok = true
         if cluster["creation_style"] != "new" && cluster["identifier"].nil?
           MU.log "CacheCluster #{cluster['name']}'s creation_style is set to #{cluster['creation_style']} but no identifier was provided. Either set creation_style to new or provide an identifier", MU::ERR
@@ -163,7 +163,9 @@ module MU
         end
         cluster["multi_az"] = true if cluster["node_count"] > 1
 
-        cluster['dependencies'] << adminFirewallRuleset(vpc: cluster['vpc'], region: cluster['region'], cloud: cluster['cloud']) if !cluster['scrub_mu_isms']
+        if !cluster['scrub_mu_isms']
+          cluster['dependencies'] << configurator.adminFirewallRuleset(vpc: cluster['vpc'], region: cluster['region'], cloud: cluster['cloud'], credentials: cluster['credentials'])
+        end
 
         ok
       end

data/modules/mu/config/cache_cluster.yml

@@ -0,0 +1,23 @@
+<% if $complexity == "complex" %>
+name: redis
+credentials: egtprod
+engine: redis
+creation_style: new
+size: cache.t2.medium
+name: memcache
+credentials: egtprod
+creation_style: new
+engine: memcached
+size: cache.t2.medium
+<% else %>
+name: redis
+credentials: egtprod
+engine: redis
+creation_style: new
+size: cache.t2.medium
+name: memcache
+credentials: egtprod
+creation_style: new
+engine: memcached
+size: cache.t2.medium
+<% end %>

data/modules/mu/config/database.rb

@@ -179,11 +179,11 @@ module MU
             },
             "create_cluster" => {
               "type" => "boolean",
-                "description" => "Rather to create a database cluster. This only applies to aurora",
+                "description" => "Create a database cluster instead of a standalone database.",
                 "default_if" => [
                   {
                     "key_is" => "engine",
-                    "value_is" => "aurora",
+                    "value_is" => "aurora-mysql",
                     "set" => true
                   }
                 ]
@@ -341,20 +341,27 @@ module MU
         # Automatically manufacture another database object, which will serve
         # as a read replica of this one, if we've set create_read_replica.
         if db['create_read_replica']
-          replica = Marshal.load(Marshal.dump(db))
-          replica['name'] = db['name']+"-replica"
-          replica['create_read_replica'] = false
-          replica['read_replica_of'] = {
-            "db_name" => db['name'],
-            "cloud" => db['cloud'],
-            "region" => db['read_replica_region'] || db['region']
-          }
-          replica['dependencies'] << {
-            "type" => "database",
-            "name" => db["name"],
-            "phase" => "groom"
-          }
-          read_replicas << replica
+          if db['create_cluster']
+            db["create_read_replica"] = false
+            MU.log "Ignoring extraneous create_read_replica flag on database cluster #{db['name']}", MU::WARN
+          else
+            replica = Marshal.load(Marshal.dump(db))
+            replica['name'] = db['name']+"-replica"
+            replica["credentials"] = db["credentials"]
+            replica['create_read_replica'] = false
+            replica["create_cluster"] = false
+            replica['read_replica_of'] = {
+              "db_name" => db['name'],
+              "cloud" => db['cloud'],
+              "region" => db['read_replica_region'] || db['region']
+            }
+            replica['dependencies'] << {
+              "type" => "database",
+              "name" => db["name"],
+              "phase" => "groom"
+            }
+            read_replicas << replica
+          end
         end
 
         # Do database cluster nodes the same way we do read replicas, by
@@ -364,7 +371,9 @@ module MU
           (1..db["cluster_node_count"]).each{ |num|
             node = Marshal.load(Marshal.dump(db))
             node["name"] = "#{db['name']}-#{num}"
+            node["credentials"] = db["credentials"]
             node["create_cluster"] = false
+            node["create_read_replica"] = false
             node["creation_style"] = "new"
             node["add_cluster_node"] = true
             node["member_of_cluster"] = {