cloud-mu 3.1.3 → 3.1.4

data/modules/mu/config/endpoint.rb

@@ -57,10 +57,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::endpoints}, bare and unvalidated.
-      # @param endpoint [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _endpoint [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(endpoint, configurator)
+      def self.validate(_endpoint, _configurator)
         ok = true
 
         ok

data/modules/mu/config/firewall_rule.rb

@@ -100,14 +100,129 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::firewall_rules}, bare and unvalidated.
-      # @param acl [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _acl [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(acl, configurator)
+      def self.validate(_acl, _configurator)
         ok = true
         ok
       end
 
     end
+
+    # FirewallRules can reference other FirewallRules, which means we need to do
+    # an extra pass to make sure we get all intra-stack dependencies correct.
+    # @param acl [Hash]: The configuration hash for the FirewallRule to check
+    # @return [Hash]
+    def resolveIntraStackFirewallRefs(acl, delay_validation = false)
+      acl["rules"].each { |acl_include|
+        if acl_include['sgs']
+          acl_include['sgs'].each { |sg_ref|
+            if haveLitterMate?(sg_ref, "firewall_rules")
+              acl["dependencies"] ||= []
+              found = false
+              acl["dependencies"].each { |dep|
+                if dep["type"] == "firewall_rule" and dep["name"] == sg_ref
+                  dep["no_create_wait"] = true
+                  found = true
+                end
+              }
+              if !found
+                acl["dependencies"] << {
+                  "type" => "firewall_rule",
+                  "name" => sg_ref,
+                  "no_create_wait" => true
+                }
+              end
+              siblingfw = haveLitterMate?(sg_ref, "firewall_rules")
+              if !siblingfw["#MU_VALIDATED"]
+# XXX raise failure somehow
+                insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation)
+              end
+            end
+          }
+        end
+      }
+      acl
+    end
+
+    # Generate configuration for the general-purpose admin firewall rulesets
+    # (security groups in AWS). Note that these are unique to regions and
+    # individual VPCs (as well as Classic, which is just a degenerate case of
+    # a VPC for our purposes.
+    # @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC).
+    # @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource.
+    # @param cloud [String]: The parent resource's cloud plugin identifier
+    # @param region [String]: Cloud provider region, if applicable.
+    # @return [Hash<String>]: A dependency description that the calling resource can then add to itself.
+    def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false)
+      if !cloud or (cloud == "AWS" and !region)
+        raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider"
+      end
+      hosts = Array.new
+      hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip
+      hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip
+      hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip
+      hosts << "#{admin_ip}/32" if admin_ip
+      hosts.uniq!
+
+      rules = []
+      if cloud == "Google"
+        rules = [
+          { "ingress" => true, "proto" => "all", "hosts" => hosts },
+          { "egress" => true, "proto" => "all", "hosts" => hosts }
+        ]
+      else
+        rules = [
+          { "proto" => "tcp", "port_range" => "0-65535", "hosts" => hosts },
+          { "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts },
+          { "proto" => "icmp", "port_range" => "-1", "hosts" => hosts }
+        ]
+      end
+
+      resclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("FirewallRule")
+
+      if rules_only
+        return rules
+      end
+
+      name = "admin"
+      name += credentials.to_s if credentials
+      realvpc = nil
+      if vpc
+        realvpc = {}
+        ['vpc_name', 'vpc_id'].each { |p|
+          if vpc[p]
+            vpc[p.sub(/^vpc_/, '')] = vpc[p] 
+            vpc.delete(p)
+          end
+        }
+        ['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field|
+          realvpc[field] = vpc[field] if !vpc[field].nil?
+        }
+        if !realvpc['id'].nil? and !realvpc['id'].empty?
+          # Stupid kludge for Google cloud_ids which are sometimes URLs and
+          # sometimes not. Requirements are inconsistent from scenario to
+          # scenario.
+          name = name + "-" + realvpc['id'].gsub(/.*\//, "")
+          realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC",  cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String)
+        elsif !realvpc['name'].nil?
+          name = name + "-" + realvpc['name']
+        end
+      end
+
+
+      acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
+      if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"]
+        acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"]
+      end
+      acl.delete("vpc") if !acl["vpc"]
+      if !resclass.isGlobal? and !region.nil? and !region.empty?
+        acl["region"] = region
+      end
+      @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl)
+      return {"type" => "firewall_rule", "name" => name}
+    end
+
   end
 end

data/modules/mu/config/folder.rb

@@ -59,10 +59,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::folder}, bare and unvalidated.
-      # @param folder [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _folder [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(folder, configurator)
+      def self.validate(_folder, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/function.rb

@@ -99,9 +99,9 @@ module MU
 
       # Generic pre-processing of {MU::Config::BasketofKittens::functions}, bare and unvalidated.
       # @param function [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(function, configurator)
+      def self.validate(function, _configurator)
         ok = true
         if !function['code']
           ok = false

data/modules/mu/config/group.rb

@@ -51,10 +51,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::group}, bare and unvalidated.
-      # @param group [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _group [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(group, configurator)
+      def self.validate(_group, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/habitat.rb

@@ -38,10 +38,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::habitat}, bare and unvalidated.
-      # @param habitat [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _habitat [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(habitat, configurator)
+      def self.validate(_habitat, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/loadbalancer.rb

@@ -383,9 +383,9 @@ module MU
 
       # Generic pre-processing of {MU::Config::BasketofKittens::loadbalancers}, bare and unvalidated.
       # @param lb [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(lb, configurator)
+      def self.validate(lb, _configurator)
         ok = true
         # Convert old-school listener declarations into target groups and health
         # checks, for which AWS and Google both have equivalents.
@@ -446,7 +446,7 @@ module MU
                 else
                   found = false
                   lb['targetgroups'].each { |tg|
-                    if l['targetgroup'] == action['targetgroup']
+                    if tg['name'] == action['targetgroup']
                       found = true
                       break
                     end

data/modules/mu/config/log.rb

@@ -36,10 +36,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::logs}, bare and unvalidated.
-      # @param log [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _log [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(log, configurator)
+      def self.validate(_log, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/msg_queue.rb

@@ -34,10 +34,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::msg_queues}, bare and unvalidated.
-      # @param queue [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _queue [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(queue, configurator)
+      def self.validate(_queue, _configurator)
         ok = true
         ok
       end

data/modules/mu/config/nosqldb.rb

@@ -35,10 +35,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::nosqldbs}, bare and unvalidated.
-      # @param db [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _db [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(db, configurator)
+      def self.validate(_db, _configurator)
         ok = true
 
         ok

data/modules/mu/config/notifier.rb

@@ -51,9 +51,9 @@ module MU
 
       # Generic pre-processing of {MU::Config::BasketofKittens::notifiers}, bare and unvalidated.
       # @param notifier [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(notifier, configurator)
+      def self.validate(notifier, _configurator)
         ok = true
 
         if notifier['subscriptions']

data/modules/mu/config/ref.rb

@@ -0,0 +1,333 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+
+  # Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
+  class Config
+
+    # A wrapper class for resources to refer to other resources, whether they
+    # be a sibling object in the current deploy, an object in another deploy,
+    # or a plain cloud id from outside of Mu.
+    class Ref
+      attr_reader :name 
+      attr_reader :type 
+      attr_reader :cloud 
+      attr_reader :deploy_id 
+      attr_reader :region 
+      attr_reader :credentials 
+      attr_reader :habitat
+      attr_reader :mommacat
+      attr_reader :tag_key 
+      attr_reader :tag_value
+      attr_reader :obj 
+
+      @@refs = []
+      @@ref_semaphore = Mutex.new
+
+      # Little bit of a factory pattern... given a hash of options for a {MU::Config::Ref} objects, first see if we have an existing one that matches our more immutable attributes (+cloud+, +id+, etc). If we do, return that. If we do not, create one, add that to our inventory, and return that instead.
+      # @param cfg [Hash]: 
+      # @return [MU::Config::Ref]
+      def self.get(cfg)
+        return cfg if cfg.is_a?(MU::Config::Ref)
+        checkfields = cfg.keys.map { |k| k.to_sym }
+        required = [:id, :type]
+
+        @@ref_semaphore.synchronize {
+          @@refs.each { |ref|
+            saw_mismatch = false
+            saw_match = false
+            needed_values = []
+            checkfields.each { |field|
+              next if !cfg[field]
+              ext_value = ref.instance_variable_get("@#{field.to_s}".to_sym)
+              if !ext_value
+                needed_values << field
+                next
+              end
+              if cfg[field] != ext_value
+                saw_mismatch = true
+              elsif required.include?(field) and cfg[field] == ext_value
+                saw_match = true
+              end
+            }
+            if saw_match and !saw_mismatch
+              # populate empty fields we got from this request
+              if needed_values.size > 0
+                newref = ref.dup
+                needed_values.each { |field|
+                  newref.instance_variable_set("@#{field.to_s}".to_sym, cfg[field])
+                  if !newref.respond_to?(field)
+                    newref.singleton_class.instance_eval { attr_reader field.to_sym }
+                  end
+                }
+                @@refs << newref
+                return newref
+              else
+                return ref
+              end
+            end
+          }
+
+        }
+
+        # if we get here, there was no match
+        newref = MU::Config::Ref.new(cfg)
+        @@ref_semaphore.synchronize {
+          @@refs << newref
+          return newref
+        }
+      end
+
+      # A way of dynamically defining +attr_reader+ without leaking memory
+      def self.define_reader(name)
+        define_method(name) {
+          instance_variable_get("@#{name.to_s}")
+        }
+      end
+
+      # @param cfg [Hash]: A Basket of Kittens configuration hash containing
+      # lookup information for a cloud object
+      def initialize(cfg)
+        cfg.keys.each { |field|
+          next if field == "tag"
+          if !cfg[field].nil?
+            self.instance_variable_set("@#{field}".to_sym, cfg[field])
+          elsif !cfg[field.to_sym].nil?
+            self.instance_variable_set("@#{field.to_s}".to_sym, cfg[field.to_sym])
+          end
+          MU::Config::Ref.define_reader(field)
+        }
+        if cfg['tag'] and cfg['tag']['key'] and
+           !cfg['tag']['key'].empty? and cfg['tag']['value']
+          @tag_key = cfg['tag']['key']
+          @tag_value = cfg['tag']['value']
+        end
+
+        if @deploy_id and !@mommacat
+          @mommacat = MU::MommaCat.getLitter(@deploy_id, set_context_to_me: false)
+        elsif @mommacat and !@deploy_id
+          @deploy_id = @mommacat.deploy_id
+        end
+
+        kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this
+      end
+
+      # Comparison operator
+      def <=>(other)
+        return 1 if other.nil?
+        self.to_s <=> other.to_s
+      end
+
+      # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}.
+      # @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+).
+      # @return [Hash]
+      def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [])
+        parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1')
+        desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource"
+        schema = {
+          "type" => "object",
+          "#MU_REFERENCE" => true,
+          "minProperties" => 1,
+          "description" => desc,
+          "properties" => {
+            "id" => {
+              "type" => "string",
+              "description" => "Cloud identifier of a resource we want to reference, typically used when leveraging resources not managed by MU"
+            },
+            "name" => {
+              "type" => "string",
+              "description" => "The short (internal Mu) name of a resource we're attempting to reference. Typically used when referring to a sibling resource elsewhere in the same deploy, or in another known Mu deploy in conjunction with +deploy_id+."
+            },
+            "type" => {
+              "type" => "string",
+              "description" => "The resource type we're attempting to reference.",
+              "enum" => MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] }
+            },
+            "deploy_id" => {
+              "type" => "string",
+              "description" => "Our target resource should be found in this Mu deploy."
+            },
+            "credentials" => MU::Config.credentials_primitive,
+            "region" => MU::Config.region_primitive,
+            "cloud" => MU::Config.cloud_primitive,
+            "tag" => {
+              "type" => "object",
+              "description" => "If the target resource supports tagging and our resource implementations +find+ method supports it, we can attempt to locate it by tag.",
+              "properties" => {
+                "key" => {
+                  "type" => "string",
+                  "description" => "The tag or label key to search against"
+                },
+                "value" => {
+                  "type" => "string",
+                  "description" => "The tag or label value to match"
+                }
+              }
+            }
+          }
+        }
+        if !["folders", "habitats"].include?(type)
+          schema["properties"]["habitat"] = MU::Config::Habitat.reference
+        end
+
+        if omit_fields
+          omit_fields.each { |f|
+            schema["properties"].delete(f)
+          }
+        end
+
+        if !type.nil?
+          schema["required"] = ["type"]
+          schema["properties"]["type"]["default"] = type
+          schema["properties"]["type"]["enum"] = [type]
+        end
+
+        aliases.each { |a|
+          a.each_pair { |k, v|
+            if schema["properties"][v]
+              schema["properties"][k] = schema["properties"][v].dup
+              schema["properties"][k]["description"] = "Alias for <tt>#{v}</tt>"
+            else
+              MU.log "Reference schema alias #{k} wants to alias #{v}, but no such attribute exists", MU::WARN, details: caller[4]
+            end
+          }
+        }
+
+        schema
+      end
+
+      # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment,
+      # of the sort that would have been used to declare this reference in the
+      # first place.
+      def to_h
+        me = { }
+
+        self.instance_variables.each { |var|
+          next if [:@obj, :@mommacat, :@tag_key, :@tag_value].include?(var)
+          val = self.instance_variable_get(var)
+          next if val.nil?
+          val = val.to_h if val.is_a?(MU::Config::Ref)
+          me[var.to_s.sub(/^@/, '')] = val
+        }
+        if @tag_key and !@tag_key.empty?
+          me['tag'] = {
+            'key' => @tag_key,
+            'value' => @tag_value
+          }
+        end
+        me
+      end
+
+      # Getter for the #{id} instance variable that attempts to populate it if
+      # it's not set.
+      # @return [String,nil]
+      def id
+        return @id if @id
+        kitten # if it's not defined, attempt to define it
+        @id
+      end
+
+      # Alias for {id}
+      # @return [String,nil]
+      def cloud_id
+        id
+      end
+
+      # Return a {MU::Cloud} object for this reference. This is only meant to be
+      # called in a live deploy, which is to say that if called during initial
+      # configuration parsing, results may be incorrect.
+      # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches.
+      def kitten(mommacat = @mommacat, shallow: false)
+        return nil if !@cloud or !@type
+
+        if @obj
+          @deploy_id ||= @obj.deploy_id
+          @id ||= @obj.cloud_id
+          @name ||= @obj.config['name']
+          return @obj
+        end
+
+        if mommacat
+          @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: false)
+          if @obj # initialize missing attributes, if we can
+            @id ||= @obj.cloud_id
+            @mommacat ||= mommacat
+            @obj.intoDeploy(@mommacat) # make real sure these are set
+            @deploy_id ||= mommacat.deploy_id
+            if !@name
+              if @obj.config and @obj.config['name']
+                @name = @obj.config['name']
+              elsif @obj.mu_name
+if @type == "folders"
+MU.log "would assign name '#{@obj.mu_name}' in ref to this folder if I were feeling aggressive", MU::WARN, details: self.to_h
+end
+#                @name = @obj.mu_name
+              end
+            end
+            return @obj
+          else
+#            MU.log "Failed to find a live '#{@type.to_s}' object named #{@name}#{@id ? " (#{@id})" : "" }#{ @habitat ? " in habitat #{@habitat}" : "" }", MU::WARN, details: self
+          end
+        end
+
+        if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id)) and !shallow
+
+          begin
+            hab_arg = if @habitat.nil?
+              [nil]
+            elsif @habitat.is_a?(MU::Config::Ref)
+              [@habitat.id]
+            elsif @habitat.is_a?(Hash)
+              [@habitat["id"]]
+            else
+              [@habitat.to_s]
+            end
+
+            found = MU::MommaCat.findStray(
+              @cloud,
+              @type,
+              name: @name,
+              cloud_id: @id,
+              deploy_id: @deploy_id,
+              region: @region,
+              habitats: hab_arg,
+              credentials: @credentials,
+              dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
+            )
+            @obj ||= found.first if found
+          rescue ThreadError => e
+            # Sometimes MommaCat calls us in a potential deadlock situation;
+            # don't be the cause of a fatal error if so, we don't need this
+            # object that badly.
+            raise e if !e.message.match(/recursive locking/)
+rescue SystemExit
+# XXX this is temporary, to cope with some debug stuff that's in findStray
+# for the nonce
+return
+          end
+        end
+
+        if @obj
+          @deploy_id ||= @obj.deploy_id
+          @id ||= @obj.cloud_id
+          @name ||= @obj.config['name']
+        end
+
+        @obj
+      end
+
+    end
+  end
+end