cloud-mu 3.1.3 → 3.3.0

data/modules/mu/config/endpoint.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/api.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/api.rb
     class Endpoint
 
       # Base configuration schema for an Endpoint (e.g. AWS API Gateway)
@@ -32,6 +32,7 @@ module MU
           "iam_role" => {"type" => "string"},
           "region" => MU::Config.region_primitive,
           "vpc" => MU::Config::VPC.reference(MU::Config::VPC::NO_SUBNETS, MU::Config::VPC::NO_NAT_OPTS),
+          "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "endpoint"),
           "methods" => {
             "type" => "array",
             "items" => {
@@ -57,10 +58,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::endpoints}, bare and unvalidated.
-      # @param endpoint [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _endpoint [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(endpoint, configurator)
+      def self.validate(_endpoint, _configurator)
         ok = true
 
         ok

data/modules/mu/config/firewall_rule.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/firewall_rule.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/firewall_rule.rb
     class FirewallRule
 
       # Base configuration schema for a FirewallRule
@@ -100,14 +100,113 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::firewall_rules}, bare and unvalidated.
-      # @param acl [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _acl [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(acl, configurator)
+      def self.validate(_acl, _configurator)
         ok = true
         ok
       end
 
     end
+
+    # FirewallRules can reference other FirewallRules, which means we need to do
+    # an extra pass to make sure we get all intra-stack dependencies correct.
+    # @param acl [Hash]: The configuration hash for the FirewallRule to check
+    # @return [Hash]
+    def resolveIntraStackFirewallRefs(acl, delay_validation = false)
+      acl["rules"].each { |acl_include|
+        if acl_include['sgs']
+          acl_include['sgs'].each { |sg_ref|
+            if haveLitterMate?(sg_ref, "firewall_rules")
+              MU::Config.addDependency(acl, sg_ref, "firewall_rule", no_create_wait: true)
+              siblingfw = haveLitterMate?(sg_ref, "firewall_rules")
+              if !siblingfw["#MU_VALIDATED"]
+# XXX raise failure somehow
+                insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation)
+              end
+            end
+          }
+        end
+      }
+      acl
+    end
+
+    # Generate configuration for the general-purpose admin firewall rulesets
+    # (security groups in AWS). Note that these are unique to regions and
+    # individual VPCs (as well as Classic, which is just a degenerate case of
+    # a VPC for our purposes.
+    # @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC).
+    # @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource.
+    # @param cloud [String]: The parent resource's cloud plugin identifier
+    # @param region [String]: Cloud provider region, if applicable.
+    # @return [Hash<String>]: A dependency description that the calling resource can then add to itself.
+    def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false)
+      if !cloud or (cloud == "AWS" and !region)
+        raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider"
+      end
+      hosts = Array.new
+      hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip
+      hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip
+      hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip
+      hosts << "#{admin_ip}/32" if admin_ip
+      hosts.uniq!
+
+      rules = []
+      if cloud == "Google"
+        rules = [
+          { "ingress" => true, "proto" => "all", "hosts" => hosts },
+          { "egress" => true, "proto" => "all", "hosts" => hosts }
+        ]
+      else
+        rules = [
+          { "proto" => "tcp", "port_range" => "0-65535", "hosts" => hosts },
+          { "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts },
+          { "proto" => "icmp", "port_range" => "-1", "hosts" => hosts }
+        ]
+      end
+
+      if rules_only
+        return rules
+      end
+
+      name = "admin"
+      name += credentials.to_s if credentials
+      realvpc = nil
+      if vpc
+        realvpc = {}
+        ['vpc_name', 'vpc_id'].each { |p|
+          if vpc[p]
+            vpc[p.sub(/^vpc_/, '')] = vpc[p] 
+            vpc.delete(p)
+          end
+        }
+        ['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field|
+          realvpc[field] = vpc[field] if !vpc[field].nil?
+        }
+        if !realvpc['id'].nil? and !realvpc['id'].empty?
+          # Stupid kludge for Google cloud_ids which are sometimes URLs and
+          # sometimes not. Requirements are inconsistent from scenario to
+          # scenario.
+          name = name + "-" + realvpc['id'].gsub(/.*\//, "")
+          realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC",  cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String)
+        elsif !realvpc['name'].nil?
+          name = name + "-" + realvpc['name']
+        end
+      end
+
+
+      acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
+      if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"]
+        acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"]
+      end
+      acl.delete("vpc") if !acl["vpc"]
+      if !MU::Cloud.resourceClass(cloud, "FirewallRule").isGlobal? and !region.nil? and !region.empty?
+        acl["region"] = region
+      end
+      @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl)
+      return {"type" => "firewall_rule", "name" => name}
+    end
+
   end
 end

data/modules/mu/config/folder.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/folder.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/folder.rb
     class Folder
 
       # Base configuration schema for a Folder
@@ -59,10 +59,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::folder}, bare and unvalidated.
-      # @param folder [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _folder [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(folder, configurator)
+      def self.validate(_folder, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/function.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/function.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/function.rb
     class Function
 
       # Base configuration schema for a Function
@@ -71,6 +71,10 @@ module MU
               "zip_file" => {
                 "type" => "string",
                 "description" => "Path to a zipped deployment package to upload."
+              }, 
+              "path" => {
+                "type" => "string",
+                "description" => "Path to a directory that can be zipped into deployment package to upload."
               } 
             }
           },
@@ -99,20 +103,25 @@ module MU
 
       # Generic pre-processing of {MU::Config::BasketofKittens::functions}, bare and unvalidated.
       # @param function [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(function, configurator)
+      def self.validate(function, _configurator)
         ok = true
         if !function['code']
           ok = false
         end
-        if function['code'] and function['code']['zip_file']
-          if !File.readable?(function['code']['zip_file'])
-            MU.log "Can't read Function deployment package #{function['code']['zip_file']}", MU::ERR
-            ok = false
-          else
-            function['code']['zip_file'] = File.realpath(File.expand_path(function['code']['zip_file']))
-          end
+
+        if function['code']
+          ['zip_file', 'path'].each { |src|
+            if function['code'][src]
+              if !File.readable?(function['code'][src]) and !Dir.exists?(function['code'][src])
+                MU.log "Function '#{function['name']}' specifies a deployment package that I can't read at #{function['code'][src]}", MU::ERR
+                ok = false
+              else
+                function['code'][src] = File.realpath(File.expand_path(function['code'][src]))
+              end
+            end
+          }
         end
 
         ok

data/modules/mu/config/group.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/group.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/group.rb
     class Group
 
       # Base configuration schema for a Group
@@ -51,10 +51,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::group}, bare and unvalidated.
-      # @param group [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _group [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(group, configurator)
+      def self.validate(_group, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/habitat.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/project.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/project.rb
     class Habitat
 
       # Base configuration schema for a Habitat
@@ -38,10 +38,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::habitat}, bare and unvalidated.
-      # @param habitat [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _habitat [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(habitat, configurator)
+      def self.validate(_habitat, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/job.rb

@@ -0,0 +1,89 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Config
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/job.rb
+    class Job
+
+      # Base configuration schema for a scheduled job
+      # @return [Hash]
+      def self.schema
+        {
+          "type" => "object",
+          "additionalProperties" => false,
+          "description" => "A cloud provider-specific facility for triggered or scheduled tasks, such as AWS CloudWatch Events or Google Cloud Scheduler.",
+          "properties" => {
+            "name" => {
+              "type" => "string"
+            },
+            "region" => MU::Config.region_primitive,
+            "credentials" => MU::Config.credentials_primitive,
+            "description" => {
+              "type" => "string",
+              "description" => "Human-readable description field for this job (this will field be overriden with the Mu deploy id on most providers unless +scrub_mu_isms+ is set)"
+            },
+            "schedule" => {
+              "type" => "object",
+              "description" => "A schedule on which to invoke this task, typically unix crontab style.",
+              "properties" => {
+                "minute" => {
+                  "type" => "string",
+                  "description" => "The minute of the hour at which to invoke this job, typically an integer between 0 and 59. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "0"
+                },
+                "hour" => {
+                  "type" => "string",
+                  "description" => "The hour at which to invoke this job, typically an integer between 0 and 23. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "day_of_month" => {
+                  "type" => "string",
+                  "description" => "The day of the month which to invoke this job, typically an integer between 1 and 31. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "month" => {
+                  "type" => "string",
+                  "description" => "The month in which to invoke this job, typically an integer between 1 and 12. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "day_of_week" => {
+                  "type" => "string",
+                  "description" => "The day of the week on which to invoke this job, typically an integer between 0 and 6. This will be validated by the cloud provider, where other more human-readable values may be supported.",
+                  "default" => "*"
+                },
+                "year" => {
+                  "type" => "string",
+                  "description" => "The year in which to invoke this job. Not honored by all cloud providers.",
+                  "default" => "*"
+                }
+              }
+            }
+          }
+        }
+      end
+
+      # Generic pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated.
+      # @param _job [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @return [Boolean]: True if validation succeeded, False otherwise
+      def self.validate(_job, _configurator)
+        ok = true
+
+        ok
+      end
+
+    end
+  end
+end

data/modules/mu/config/loadbalancer.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/loadbalancer.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/loadbalancer.rb
     class LoadBalancer
 
       # Generate schema for a LoadBalancer health check
@@ -64,6 +64,45 @@ module MU
         }
       end
 
+      # Generate schema for a LoadBalancer redirect
+      # @return [Hash]
+      def self.redirect
+        {
+          "type" => "object",
+          "title" => "redirect",
+          "additionalProperties" => false,
+          "description" => "Instruct our LoadBalancer to redirect traffic to another host, port, and/or path.",
+          "properties" => {
+            "protocol" => {
+              "type" => "string",
+              "default" => "HTTPS"
+            },
+            "port" => {
+              "type" => "integer",
+              "default" => 443
+            },
+            "host" => {
+              "type" => "string",
+              "default" => "\#{host}"
+            },
+            "path" => {
+              "type" => "string",
+              "default" => "/\#{path}"
+            },
+            "query" => {
+              "type" => "string",
+              "default" => "\#{query}"
+            },
+            "status_code" => {
+              "type" => "integer",
+              "description" => "The HTTP status code when issuing a redirect",
+              "default" => 301,
+              "enum" => [301, 302]
+            },
+          }
+        }
+      end
+
       # Base configuration schema for a LoadBalancer
       # @return [Hash]
       def self.schema
@@ -261,7 +300,7 @@ module MU
               "type" => "array",
               "items" => {
                 "type" => "object",
-                "required" => ["lb_protocol", "lb_port", "instance_protocol", "instance_port"],
+                "required" => ["lb_protocol", "lb_port"],
                 "additionalProperties" => false,
                 "description" => "A list of port/protocols which this Load Balancer should answer.",
                 "properties" => {
@@ -279,6 +318,7 @@ module MU
                     "enum" => ["HTTP", "HTTPS", "TCP", "SSL", "UDP"],
                     "description" => "Specifies the load balancer transport protocol to use for routing - HTTP, HTTPS, TCP, SSL, or UDP. SSL and UDP are only valid in Google Cloud."
                   },
+                  "redirect" => MU::Config::LoadBalancer.redirect,
                   "targetgroup" => {
                     "type" => "string",
                     "description" => "Which of our declared targetgroups should be the back-end for this listener's traffic"
@@ -309,14 +349,14 @@ module MU
                     "items" => {
                       "type" => "object",
                       "description" => "Rules to route requests to different target groups based on the request path",
-                      "required" => ["conditions", "order"],
+                      "required" => ["order", "conditions"],
                       "additionalProperties" => false,
                       "properties" => {
                         "conditions" => {
                           "type" => "array",
                           "items" => {
                             "type" => "object",
-                            "description" => "Rule condition",
+                            "description" => "Rule conditionl; if none are specified (or if none match) the default action will be set.",
                             "required" => ["field", "values"],
                             "additionalProperties" => false,
                             "properties" => {
@@ -339,16 +379,17 @@ module MU
                           "type" => "array",
                           "items" => {
                             "type" => "object",
-                            "description" => "Rule action",
-                            "required" => ["action", "targetgroup"],
+                            "description" => "Rule action, which must specify one of +targetgroup+ or +redirect+",
+                            "required" => ["action"],
                             "additionalProperties" => false,
                             "properties" => {
                               "action" => {
                                 "type" => "string",
                                 "default" => "forward",
                                 "description" => "An action to take when a match occurs. Currently, only forwarding to a targetgroup is supported.",
-                                "enum" => ["forward"]
+                                "enum" => ["forward", "redirect"]
                               },
+                              "redirect" => MU::Config::LoadBalancer.redirect,
                               "targetgroup" => {
                                 "type" => "string",
                                 "description" => "Which of our declared targetgroups should be the recipient of this traffic. If left unspecified, will default to the default targetgroup of this listener."
@@ -383,9 +424,9 @@ module MU
 
       # Generic pre-processing of {MU::Config::BasketofKittens::loadbalancers}, bare and unvalidated.
       # @param lb [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(lb, configurator)
+      def self.validate(lb, _configurator)
         ok = true
         # Convert old-school listener declarations into target groups and health
         # checks, for which AWS and Google both have equivalents.
@@ -405,13 +446,18 @@ module MU
               "proto" => l["instance_protocol"],
               "port" => l["instance_port"]
             }
-            if lb["healthcheck"]
-              hc_target = lb['healthcheck']['target'].match(/^([^:]+):(\d+)(.*)/)
-              tg["healthcheck"] = lb['healthcheck'].dup
+            if l["redirect"]
+              tg["proto"] ||= l["redirect"]["protocol"]
+              tg["port"] ||= l["redirect"]["port"]
+            end
+            l['healthcheck'] ||= lb['healthcheck'] if lb['healthcheck']
+            if l["healthcheck"]
+              hc_target = l['healthcheck']['target'].match(/^([^:]+):(\d+)(.*)/)
+              tg["healthcheck"] = l['healthcheck'].dup
               proto = ["HTTP", "HTTPS"].include?(hc_target[1]) ? hc_target[1] : l["instance_protocol"]
               tg['healthcheck']['target'] = "#{proto}:#{hc_target[2]}#{hc_target[3]}"
               tg['healthcheck']["httpcode"] = "200,301,302"
-              MU.log "Converting classic-style ELB health check target #{lb['healthcheck']['target']} to ALB style for target group #{tgname} (#{l["instance_protocol"]}:#{l["instance_port"]}).", details: tg['healthcheck']
+              MU.log "Converting classic-style ELB health check target #{l['healthcheck']['target']} to ALB style for target group #{tgname} (#{l["instance_protocol"]}:#{l["instance_port"]}).", details: tg['healthcheck']
             end
             lb["targetgroups"] << tg
           }
@@ -446,7 +492,7 @@ module MU
                 else
                   found = false
                   lb['targetgroups'].each { |tg|
-                    if l['targetgroup'] == action['targetgroup']
+                    if tg['name'] == action['targetgroup']
                       found = true
                       break
                     end