cloud-mu 3.2.0 → 3.5.0

data/modules/mu/config/bucket.rb

@@ -50,6 +50,24 @@ module MU
               "default" => "index.html",
               "description" => "If +web_enabled+, return this object when \"diretory\" (a path not ending in a key/object) is invoked."
             },
+            "upload" => {
+              "type" => "array",
+              "items" => {
+                "type" => "object",
+                "description" => "Upload objects to a bucket, where supported",
+                "required" => ["source", "destination"],
+                "properties" => {
+                  "source" => {
+                    "type" => "string",
+                    "description" => "A file or directory to upload. If a directory is specified, it will be recursively mirrored to the bucket +destination+."
+                  },
+                  "destination" => {
+                    "type" => "string",
+                    "description" => "Path to which +source+ file(s) will be uploaded in the bucket, relative to +/+"
+                  }
+                }
+              }
+            },
             "policies" => {
               "type" => "array",
               "items" => MU::Config::Role.policy_primitive(subobjects: true, grant_to: true, permissions_optional: true, targets_optional: true)
@@ -59,12 +77,23 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::buckets}, bare and unvalidated.
-      # @param _bucket [Hash]: The resource to process and validate
+      # @param bucket [Hash]: The resource to process and validate
       # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(_bucket, _configurator)
+      def self.validate(bucket, _configurator)
         ok = true
 
+        if bucket['upload']
+          bucket['upload'].each { |batch|
+            if !File.exists?(batch['source'])
+              MU.log "Bucket '#{bucket['name']}' specifies upload for file/directory that does not exist", MU::ERR, details: batch
+              ok = false
+              next
+            end
+            batch['source'] = File.realpath(File.expand_path(batch['source']))
+          }
+        end
+
         ok
       end
 

data/modules/mu/config/cache_cluster.rb

@@ -54,7 +54,7 @@ module MU
                 "type" => "string",
                 "default" => "redis"
             },
-            "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true),
+            "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "cache_cluster"),
             "dns_sync_wait" => {
                 "type" => "boolean",
                 "description" => "Wait for DNS record to propagate in DNS Zone.",

data/modules/mu/config/cdn.rb

@@ -0,0 +1,100 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Config
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/job.rb
+    class CDN
+
+      # Base configuration schema for a scheduled job
+      # @return [Hash]
+      def self.schema
+        {
+          "type" => "object",
+          "additionalProperties" => false,
+          "required" => ["origins"],
+          "properties" => {
+            "name" => {
+              "type" => "string"
+            },
+            "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "cdn"),
+            "default_object" => {
+              "type" => "string",
+              "default" => "index.html"
+            },
+            "credentials" => MU::Config.credentials_primitive,
+            "aliases" => {
+              "type" => "array",
+              "items" => {
+                "type" => "string"
+              }
+            },
+            "origins" => {
+              "type" => "array",
+              "minItems" => 1,
+              "items" => {
+                "type" => "object",
+                "description" => "One or more back-end sources which this CDN will cache",
+                "required" => ["name"],
+                "properties" => {
+                  "name" => {
+                    "type" => "string",
+                    "description" => "A unique identifying string which other components of this distribution may use to reference this origin"
+                  },
+                  "domain_name" => {
+                    "type" => "string",
+                    "description" => "Domain name of the back-end web server or other resource behind this CDN"
+                  },
+                  "path" => {
+                    "type" => "string",
+                    "default" => "",
+                    "description" => "Optional path on back-end service to which to map front-end requests"
+                  }
+                }
+              }
+            },
+            "behaviors" => {
+              "type" => "array",
+              "items" => {
+                "description" => "Customize the behavior of requests sent to one of this CDN's configured +origins+",
+                "type" => "object",
+                "properties" => {
+                  "origin" => {
+                    "type" => "string",
+                    "description" => "Which of our +origins+ this set of behaviors should map to, by its +name+ field."
+                  },
+                  "path_pattern" => {
+                    "type" => "string",
+                    "description" => "The request path or paths for which this behavior should be invoked"
+                  }
+                }
+              }
+            }
+          }
+        }
+      end
+
+      # Generic pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated.
+      # @param _job [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @return [Boolean]: True if validation succeeded, False otherwise
+      def self.validate(_job, _configurator)
+        ok = true
+
+        ok
+      end
+
+    end
+  end
+end

data/modules/mu/config/container_cluster.rb

@@ -48,7 +48,7 @@ module MU
               "properties" => {
                 "version" => {
                   "type" => "string",
-                  "default" => "1.14",
+                  "default" => "1.15",
                   "description" => "Version of Kubernetes control plane to deploy",
                 },
                 "max_pods" => {

data/modules/mu/config/database.rb

@@ -62,7 +62,7 @@ module MU
               "default" => false
             },
             "member_of_cluster" => MU::Config::Ref.schema(type: "databases", desc: "Internal use"),
-            "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true),
+            "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "database"),
             "dns_sync_wait" => {
                 "type" => "boolean",
                 "description" => "Wait for DNS record to propagate in DNS Zone.",
@@ -341,7 +341,7 @@ module MU
               "region" => db['region'],
               "credentials" => db['credentials'],
             }
-            MU::Config.addDependency(replica, db["name"], "database", phase: "groom")
+            MU::Config.addDependency(replica, db["name"], "database", their_phase: "groom")
             read_replicas << replica
           end
         end
@@ -367,7 +367,7 @@ module MU
               "type" => "databases"
             }
             # AWS will figure out for us which database instance is the writer/master so we can create all of them concurrently.
-            MU::Config.addDependency(node, db["name"], "database", phase: "groom")
+            MU::Config.addDependency(node, db["name"], "database", their_phase: "groom")
             cluster_nodes << node
 
            # Alarms are set on each DB cluster node, not on the cluster itself,

data/modules/mu/config/dnszone.rb

@@ -60,7 +60,7 @@ module MU
       # @param default_type [String]: The type of record to make default (e.g. An, CNAME, etc)
       # @param need_zone [Boolean]: Whether to explicitly require a zone be declared
       # @return [Hash]
-      def self.records_primitive(need_target: true, default_type: nil, need_zone: false)
+      def self.records_primitive(need_target: true, default_type: nil, need_zone: false, embedded_type: nil)
         dns_records_primitive = {
           "type" => "array",
           "maxItems" => 100,
@@ -107,8 +107,9 @@ module MU
                   },
                   "mu_type" => {
                     "type" => "string",
-                    "description" => "The Mu resource type to search the deployment for.",
-                      "enum" => ["loadbalancer", "server", "database", "cache_cluster"]
+                    "description" => "The mu type of a resource being targeted.",
+                    "enum" => embedded_type ? [embedded_type] : ["loadbalancer", "server", "database", "cache_cluster", "endpoint", "cdn"],
+                    "default" => embedded_type
                   },
                   "target_type" => {
                       "description" => "If the target is a public or a private resource. This only applies to servers/server_pools when using automatic DNS registration. If set to public but the target only has a private address, the private address will be used",

data/modules/mu/config/endpoint.rb

@@ -32,6 +32,7 @@ module MU
           "iam_role" => {"type" => "string"},
           "region" => MU::Config.region_primitive,
           "vpc" => MU::Config::VPC.reference(MU::Config::VPC::NO_SUBNETS, MU::Config::VPC::NO_NAT_OPTS),
+          "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "endpoint"),
           "methods" => {
             "type" => "array",
             "items" => {

data/modules/mu/config/firewall_rule.rb

@@ -119,7 +119,7 @@ module MU
         if acl_include['sgs']
           acl_include['sgs'].each { |sg_ref|
             if haveLitterMate?(sg_ref, "firewall_rules")
-              MU::Config.addDependency(acl, sg_ref, "firewall_rule", no_create_wait: true)
+              MU::Config.addDependency(acl, sg_ref, "firewall_rule", my_phase: "groom")
               siblingfw = haveLitterMate?(sg_ref, "firewall_rules")
               if !siblingfw["#MU_VALIDATED"]
 # XXX raise failure somehow

data/modules/mu/config/function.rb

@@ -71,6 +71,10 @@ module MU
               "zip_file" => {
                 "type" => "string",
                 "description" => "Path to a zipped deployment package to upload."
+              }, 
+              "path" => {
+                "type" => "string",
+                "description" => "Path to a directory that can be zipped into deployment package to upload."
               } 
             }
           },
@@ -106,13 +110,18 @@ module MU
         if !function['code']
           ok = false
         end
-        if function['code'] and function['code']['zip_file']
-          if !File.readable?(function['code']['zip_file'])
-            MU.log "Can't read Function deployment package #{function['code']['zip_file']}", MU::ERR
-            ok = false
-          else
-            function['code']['zip_file'] = File.realpath(File.expand_path(function['code']['zip_file']))
-          end
+
+        if function['code']
+          ['zip_file', 'path'].each { |src|
+            if function['code'][src]
+              if !File.readable?(function['code'][src]) and !Dir.exists?(function['code'][src])
+                MU.log "Function '#{function['name']}' specifies a deployment package that I can't read at #{function['code'][src]}", MU::ERR
+                ok = false
+              else
+                function['code'][src] = File.realpath(File.expand_path(function['code'][src]))
+              end
+            end
+          }
         end
 
         ok

data/modules/mu/config/job.rb

@@ -0,0 +1,89 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Config
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/job.rb
+    class Job
+
+      # Base configuration schema for a scheduled job
+      # @return [Hash]
+      def self.schema
+        {
+          "type" => "object",
+          "additionalProperties" => false,
+          "description" => "A cloud provider-specific facility for triggered or scheduled tasks, such as AWS CloudWatch Events or Google Cloud Scheduler.",
+          "properties" => {
+            "name" => {
+              "type" => "string"
+            },
+            "region" => MU::Config.region_primitive,
+            "credentials" => MU::Config.credentials_primitive,
+            "description" => {
+              "type" => "string",
+              "description" => "Human-readable description field for this job (this will field be overriden with the Mu deploy id on most providers unless +scrub_mu_isms+ is set)"
+            },
+            "schedule" => {
+              "type" => "object",
+              "description" => "A schedule on which to invoke this task, typically unix crontab style.",
+              "properties" => {
+                "minute" => {
+                  "type" => "string",
+                  "description" => "The minute of the hour at which to invoke this job, typically an integer between 0 and 59. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "0"
+                },
+                "hour" => {
+                  "type" => "string",
+                  "description" => "The hour at which to invoke this job, typically an integer between 0 and 23. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "day_of_month" => {
+                  "type" => "string",
+                  "description" => "The day of the month which to invoke this job, typically an integer between 1 and 31. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "month" => {
+                  "type" => "string",
+                  "description" => "The month in which to invoke this job, typically an integer between 1 and 12. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "day_of_week" => {
+                  "type" => "string",
+                  "description" => "The day of the week on which to invoke this job, typically an integer between 0 and 6. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "year" => {
+                  "type" => "string",
+                  "description" => "The year in which to invoke this job. Not honored by all cloud providers.",
+                  "default" => "*"
+                }
+              }
+            }
+          }
+        }
+      end
+
+      # Generic pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated.
+      # @param _job [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @return [Boolean]: True if validation succeeded, False otherwise
+      def self.validate(_job, _configurator)
+        ok = true
+
+        ok
+      end
+
+    end
+  end
+end

data/modules/mu/config/notifier.rb

@@ -36,12 +36,12 @@ module MU
               "items" => {
                 "type" => "object",
                 "description" => "A list of people or resources which should receive notifications",
-                "required" => ["endpoint"],
                 "properties" => {
                   "endpoint" => {
                     "type" => "string",
-                    "description" => "The endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number."
-                  }
+                    "description" => "Shorthand for an endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number. For complex cases, such as referencing an AWS Lambda function defined elsewhere in your Mu stack, use +resource+ instead."
+                  },
+                  "resource" => MU::Config::Ref.schema(desc: "A cloud resource that is a valid notification target for this notifier. For simple use cases, such as external email addresses or SMS, use +endpoint+ instead.")
                 }
               }
             }
@@ -56,23 +56,12 @@ module MU
       def self.validate(notifier, _configurator)
         ok = true
 
+
         if notifier['subscriptions']
           notifier['subscriptions'].each { |sub|
-            if !sub["type"]
-              if sub["endpoint"].match(/^http:/i)
-                sub["type"] = "http"
-              elsif sub["endpoint"].match(/^https:/i)
-                sub["type"] = "https"
-              elsif sub["endpoint"].match(/^sqs:/i)
-                sub["type"] = "sqs"
-              elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
-                sub["type"] = "sms"
-              elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
-                sub["type"] = "email"
-              else
-                MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR
-                ok = false
-              end
+            if !sub['endpoint'] and !sub['resource']
+              MU.log "Notifier '#{notifier['name']}' must specify either resource or endpoint in subscription", MU::ERR, details: sub
+              ok = false
             end
           }
         end

data/modules/mu/config/ref.rb

@@ -121,6 +121,10 @@ module MU
           @deploy_id = @mommacat.deploy_id
         end
 
+        # canonicalize the 'type' argument
+        _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        @type = cfg_plural if cfg_plural
+
         kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this
       end
 
@@ -140,6 +144,13 @@ module MU
         end
       end
 
+      # Lets callers set attributes like a {Hash}
+      # @param attribute [String,Symbol]
+      def []=(attribute, value)
+        instance_variable_set("@#{attribute.to_s}".to_sym, value)
+        self.class.define_reader(attribute)
+      end
+
       # Unset an attribute. Sort of. We can't actually do that, so nil it out
       # and we get the behavior we want.
       def delete(attribute)
@@ -150,7 +161,7 @@ module MU
       # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}.
       # @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+).
       # @return [Hash]
-      def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [])
+      def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [], any_type: false)
         parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1')
         desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource"
         schema = {
@@ -205,7 +216,9 @@ module MU
           }
         end
 
-        if !type.nil?
+        if any_type
+          schema["properties"]["type"].delete("enum")
+        elsif !type.nil?
           schema["required"] = ["type"]
           schema["properties"]["type"]["default"] = type
           schema["properties"]["type"]["enum"] = [type]
@@ -225,6 +238,13 @@ module MU
         schema
       end
 
+      # Is our +@type+ attribute a Mu-supported type, or some rando string?
+      # @return [Boolean]
+      def is_mu_type?
+        _shortclass, _cfg_name, type, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        !type.nil?
+      end
+
       # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment,
       # of the sort that would have been used to declare this reference in the
       # first place.
@@ -266,10 +286,23 @@ module MU
       # called in a live deploy, which is to say that if called during initial
       # configuration parsing, results may be incorrect.
       # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches.
-      def kitten(mommacat = @mommacat, shallow: false, debug: false)
-        return nil if !@cloud or !@type
+      def kitten(mommacat = @mommacat, shallow: false, debug: false, cloud: nil)
+        cloud ||= @cloud
+        return nil if !cloud or !@type
+
+        _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false)
+        if cfg_plural
+          @type = cfg_plural # make sure this is the thing we expect
+        else
+          return nil # we don't do non-muish resources
+        end
+
         loglevel = debug ? MU::NOTICE : MU::DEBUG
 
+        if debug
+          MU.log "this mf kitten", MU::WARN, details: caller
+        end
+
         if @obj
           @deploy_id ||= @obj.deploy_id
           @id ||= @obj.cloud_id
@@ -277,7 +310,7 @@ module MU
           return @obj
         end
 
-        if mommacat and !caller.grep(/`findLitterMate'/) # XXX the dumbest
+        if mommacat and caller.grep(/`findLitterMate'/).empty? # XXX the dumbest
           MU.log "Looking for #{@type} #{@name} #{@id} in deploy #{mommacat.deploy_id}", loglevel
           begin
             @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: debug)
@@ -309,7 +342,7 @@ end
           end
         end
 
-        if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(@id)) and !shallow
+        if !@obj and !(cloud == "Google" and @id and @type == "users" and MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(@id)) and !shallow
           try_deploy_id = @deploy_id
 
           begin
@@ -323,8 +356,20 @@ end
               [@habitat.to_s]
             end
 
+            MU.log "Ref#kitten calling findStray", loglevel, details: {
+              cloud: cloud,
+              type: @type,
+              name: @name,
+              cloud_id: @id,
+              deploy_id: try_deploy_id,
+              region: @region,
+              habitats: hab_arg,
+              credentials: @credentials,
+              dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type) or @id)
+            }
+
             found = MU::MommaCat.findStray(
-              @cloud,
+              cloud,
               @type,
               name: @name,
               cloud_id: @id,
@@ -332,12 +377,13 @@ end
               region: @region,
               habitats: hab_arg,
               credentials: @credentials,
-              dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
+              dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type) or @id)
             )
+            MU.log "Ref#kitten results from findStray", loglevel, details: found
             @obj ||= found.first if found
           rescue MU::MommaCat::MultipleMatches => e
             if try_deploy_id.nil? and MU.deploy_id
-              MU.log "Attempting to narrow down #{@cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE
+              MU.log "Attempting to narrow down #{cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE
               try_deploy_id = MU.deploy_id
               retry
             else