cloud-mu 3.1.4 → 3.3.1

data/modules/mu/{clouds → providers}/aws/firewall_rule.rb

@@ -18,7 +18,7 @@ module MU
     class AWS
       # A firewall ruleset as configured in {MU::Config::BasketofKittens::firewall_rules}
       class FirewallRule < MU::Cloud::FirewallRule
-        require "mu/clouds/aws/vpc"
+        require "mu/providers/aws/vpc"
 
         @admin_sgs = Hash.new
         @admin_sg_semaphore = Mutex.new
@@ -381,24 +381,24 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          filters = nil
-          if flags and flags["vpc_id"]
-            filters = [
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          filters = if flags and flags["vpc_id"]
+            [
               {name: "vpc-id", values: [flags["vpc_id"]]}
             ]
           else
             filters = [
-              {name: "tag:MU-ID", values: [MU.deploy_id]}
+              {name: "tag:MU-ID", values: [deploy_id]}
             ]
             if !ignoremaster
               filters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
             end
+            filters
           end
 
           # Some services create sneaky rogue ENIs which then block removal of
           # associated security groups. Find them and fry them.
-          MU::Cloud::AWS::VPC.purge_interfaces(noop, filters, region: region, credentials: credentials)
+          MU::Cloud.resourceClass("AWS", "VPC").purge_interfaces(noop, filters, region: region, credentials: credentials)
 
           resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
             filters: filters
@@ -408,134 +408,134 @@ module MU
             MU.log "Revoking rules in EC2 Security Group #{sg.group_name} (#{sg.group_id})"
 
             if !noop
-              ingress_to_revoke = Array.new
-              egress_to_revoke = Array.new
-              sg.ip_permissions.each { |hole|
-                ingress_to_revoke << MU.structToHash(hole)
-                ingress_to_revoke.each { |rule|
-                  if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs] .size == 0
-                    rule.delete(:user_id_group_pairs)
-                  elsif !rule[:user_id_group_pairs].nil?
-                    rule[:user_id_group_pairs].each { |group_ref|
-                      group_ref = MU.structToHash(group_ref)
-                      group_ref.delete(:group_name) if group_ref[:group_id]
-                    }
-                  end
+              revoke_rules(sg, region: region, credentials: credentials)
+              revoke_rules(sg, egress: true, region: region, credentials: credentials)
+            end
+          }
 
-                  if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
-                    rule.delete(:ip_ranges)
-                  end
+          resp.data.security_groups.each { |sg|
+            next if sg.group_name == "default"
+            MU.log "Removing EC2 Security Group #{sg.group_name}"
 
-                  if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
-                    rule.delete(:prefix_list_ids)
-                  end
-                  
-                  if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
-                    rule.delete(:ipv_6_ranges)
-                  end
-                }
-              }
-              sg.ip_permissions_egress.each { |hole|
-                egress_to_revoke << MU.structToHash(hole)
-                egress_to_revoke.each { |rule|
-                  if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs].size == 0
-                    rule.delete(:user_id_group_pairs)
-                  elsif !rule[:user_id_group_pairs].nil?
-                    rule[:user_id_group_pairs].each { |group_ref|
-                      group_ref = MU.structToHash(group_ref)
-                      group_ref.delete(:group_name) if group_ref[:group_id]
+            on_retry = Proc.new {
+              # try to get out from under loose network interfaces with which
+              # we're associated
+              if sg.vpc_id
+                default_sg = MU::Cloud.resourceClass("AWS", "VPC").getDefaultSg(sg.vpc_id, region: region, credentials: credentials)
+                if default_sg
+                  eni_resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
+                    filters: [ {name: "group-id", values: [sg.group_id]} ]
+                  )
+                  if eni_resp and eni_resp.data and
+                     eni_resp.data.network_interfaces
+                    eni_resp.data.network_interfaces.each { |iface|
+                      iface_groups = iface.groups.map { |if_sg| if_sg.group_id }
+                      iface_groups.delete(sg.group_id)
+                      iface_groups << default_sg if iface_groups.empty?
+                      MU.log "Attempting to remove #{sg.group_id} (#{sg.group_name}) from ENI #{iface.network_interface_id}"
+                      begin
+                        MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
+                          network_interface_id: iface.network_interface_id,
+                          groups: iface_groups
+                        )
+                      rescue ::Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound
+                        # fine by me
+                      rescue ::Aws::EC2::Errors::AuthFailure
+                        MU.log "Permission denied attempting to trim Security Group list for #{iface.network_interface_id}", MU::WARN, details: iface.groups.map { |g| g.group_name }.join(",")+" => default"
+                      end
                     }
                   end
+                end
+              end
+            }
 
-                  if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
-                    rule.delete(:ip_ranges)
-                  end
+            if !noop
+              MU.retrier([Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse], ignoreme: [Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10, on_retry: on_retry) {
+                begin
+                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id)
+                rescue Aws::EC2::Errors::CannotDelete => e
+                  MU.log e.message, MU::WARN
+                end
+              }
+            end
 
-                  if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
-                    rule.delete(:prefix_list_ids)
-                  end
+          }
+        end
 
-                  if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
-                    rule.delete(:ipv_6_ranges)
-                  end
+        def self.revoke_rules(sg, egress: false, region: MU.myregion, credentials: nil)
+          holes = sg.send(egress ? :ip_permissions_egress : :ip_permissions)
+
+          to_revoke = []
+
+          holes.each { |hole|
+            to_revoke << MU.structToHash(hole)
+            to_revoke.each { |rule|
+              if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs].size == 0
+                rule.delete(:user_id_group_pairs)
+              elsif !rule[:user_id_group_pairs].nil?
+                rule[:user_id_group_pairs].each { |group_ref|
+                  group_ref = MU.structToHash(group_ref)
+                  group_ref.delete(:group_name) if group_ref[:group_id]
                 }
-              }
-              begin
+              end
 
-                if ingress_to_revoke.size > 0
-                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
-                      group_id: sg.group_id,
-                      ip_permissions: ingress_to_revoke
-                  )
-                end
-                if egress_to_revoke.size > 0
-                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
-                      group_id: sg.group_id,
-                      ip_permissions: egress_to_revoke
-                  )
-                end
-              rescue Aws::EC2::Errors::InvalidPermissionNotFound
-                MU.log "Rule in #{sg.group_id} disappeared before I could remove it", MU::WARN
+              if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
+                rule.delete(:ip_ranges)
               end
-            end
-          }
 
-          resp.data.security_groups.each { |sg|
-            next if sg.group_name == "default"
-            MU.log "Removing EC2 Security Group #{sg.group_name}"
+              if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
+                rule.delete(:prefix_list_ids)
+              end
 
-            retries = 0
-            begin
-              MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id) if !noop
-            rescue Aws::EC2::Errors::CannotDelete => e
-              MU.log e.message, MU::WARN
-            rescue Aws::EC2::Errors::InvalidGroupNotFound
-              MU.log "EC2 Security Group #{sg.group_name} disappeared before I could delete it!", MU::WARN
-            rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
-              if retries < 10
-                MU.log "EC2 Security Group #{sg.group_name} is still in use, waiting...", MU::NOTICE
-                # try to get out from under loose network interfaces with which
-                # we're associated
-                if sg.vpc_id
-                  # get the default SG for this VPC
-                  default_resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
-                    filters: [
-                      { name: "group-name", values: ["default"] },
-                      { name: "vpc-id", values: [sg.vpc_id] }
-                    ]
-                  ).security_groups
-                  if default_resp and default_resp.size == 1
-                    default_sg = default_resp.first.group_id
-                    eni_resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
-                      filters: [ {name: "group-id", values: [sg.group_id]} ]
-                    )
-                    if eni_resp and eni_resp.data and
-                       eni_resp.data.network_interfaces
-                      eni_resp.data.network_interfaces.each { |iface|
-                        iface_groups = iface.groups.map { |if_sg| if_sg.group_id }
-                        iface_groups.delete(sg.group_id)
-                        iface_groups << default_sg if iface_groups.empty?
-                        MU.log "Attempting to remove #{sg.group_id} (#{sg.group_name}) from ENI #{iface.network_interface_id}"
-                        begin
-                          MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
-                            network_interface_id: iface.network_interface_id,
-                            groups: iface_groups
-                          )
-                        rescue ::Aws::EC2::Errors::AuthFailure
-                          MU.log "Permission denied attempting to trim Security Group list for #{iface.network_interface_id}", MU::WARN, details: iface.groups.map { |g| g.group_name }.join(",")+" => default"
-                        end
-                      }
-                    end
-                  end
-                end
+              if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
+                rule.delete(:ipv_6_ranges)
+              end
+            }
+          }
 
-                sleep 10
-                retries = retries + 1
-                retry
+          if to_revoke.size > 0
+            begin
+              if egress
+                MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
+                  group_id: sg.group_id,
+                  ip_permissions: to_revoke
+                )
               else
-                MU.log "Failed to delete #{sg.group_name}", MU::ERR
+                MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
+                  group_id: sg.group_id,
+                  ip_permissions: to_revoke
+                )
               end
+            rescue Aws::EC2::Errors::InvalidPermissionNotFound
+              MU.log "Rule in #{sg.group_id} disappeared before I could remove it", MU::WARN
             end
+          end
+
+        end
+        private_class_method :revoke_rules
+
+        # Return an AWS-specific chunk of schema commonly used in the +ingress_rules+ parameter of other resource types.
+        # @return [Hash]
+        def self.ingressRuleAddtlSchema
+          {
+            "items" => {
+              "properties" => {
+                "sgs" => {
+                  "type" => "array",
+                  "items" => {
+                    "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
+                    "type" => "string"
+                  }
+                },
+                "lbs" => {
+                  "type" => "array",
+                  "items" => {
+                    "description" => "AWS Load Balancers which will have this rule applied to their traffic",
+                    "type" => "string"
+                  }
+                }
+              }
+            }
           }
         end
 
@@ -648,36 +648,16 @@ module MU
 
             if rule['firewall_rules']
               rule['firewall_rules'].each { |sg|
-                if sg.is_a?(MU::Config::Ref) and sg.name
-  	              acl["dependencies"] << {
-    	              "type" => "firewall_rule",
-      	            "name" => sg.name,
-                    "no_create_wait" => true
-	                }
-                elsif sg['name'] and !sg['deploy_id']
-  	              acl["dependencies"] << {
-    	              "type" => "firewall_rule",
-      	            "name" => sg['name'],
-                    "no_create_wait" => true
-	                }
+                if sg['name'] and !sg['deploy_id']
+                  MU::Config.addDependency(acl, sg['name'], "firewall_rule", no_create_wait: true)
                 end
               }
             end
 
             if rule['loadbalancers']
               rule['loadbalancers'].each { |lb|
-                if lb.is_a?(MU::Config::Ref) and lb.name
-  	              acl["dependencies"] << {
-    	              "type" => "loadbalancer",
-      	            "name" => lb.name,
-                    "phase" => "groom"
-	                }
-                elsif lb['name'] and !lb['deploy_id']
-  	              acl["dependencies"] << {
-                    "type" => "loadbalancer",
-                    "name" => lb['name'],
-                    "phase" => "groom"
-	                }
+                if lb['name'] and !lb['deploy_id']
+                  MU::Config.addDependency(acl, lb['name'], "loadbalancer", phase: "groom")
                 end
               }
             end
@@ -719,32 +699,7 @@ module MU
 
         private
 
-        #########################################################################
-        # Manufacture an EC2 security group. The second parameter, rules, is an
-        # "ingress_rules" structure parsed and validated by MU::Config.
-        #########################################################################
-        def setRules(rules, add_to_self: false, ingress: true, egress: false)
-          describe
-          # XXX warn about attempt to set rules before we exist
-          return if rules.nil? or rules.size == 0 or !@cloud_id
-
-          # add_to_self means that this security is a "member" of its own rules
-          # (which is to say, objects that have this SG are allowed in my these
-          # rules)
-          if add_to_self
-            rules.each { |rule|
-              if rule['sgs'].nil? or !rule['sgs'].include?(@cloud_id)
-                new_rule = rule.clone
-                new_rule.delete('hosts')
-                rule['sgs'] = Array.new if rule['sgs'].nil?
-                rule['sgs'] << @cloud_id
-              end
-            }
-          end
-
-          ec2_rules = convertToEc2(rules)
-          ext_permissions = MU.structToHash(cloud_desc.ip_permissions)
-
+        def purge_extraneous_rules(ec2_rules, ext_permissions)
           # Purge any old rules that we're sure we created (check the comment)
           # but which are no longer configured.
           ext_permissions.each { |ext_rule|
@@ -781,97 +736,109 @@ module MU
                 ip_permissions: [ext_rule]
               )
             end
-
           }
+        end
 
-          # Creating an empty security group is ok, so don't freak out if we get
-          # a null rule list.
-          if !ec2_rules.nil?
-            ec2_rules.uniq!
-            retries = 0
-            ec2_rules.each { |rule|
-              haverule = nil
-              different = false
-              ext_permissions.each { |ext_rule|
-                if rule[:from_port] == ext_rule[:from_port] and
-                   rule[:to_port] == ext_rule[:to_port] and
-                   rule[:ip_protocol] == ext_rule[:ip_protocol]
-                  haverule = ext_rule
-                  ext_rule.keys.each { |k|
-                    if ext_rule[k].nil? or ext_rule[k] == []
-                      haverule.delete(k)
-                    end
-                    different = true if rule[k] != ext_rule[k]
-                  }
-                  break
-                end
-              }
-              if haverule and !different
-                MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
-                next
+        #########################################################################
+        # Manufacture an EC2 security group. The second parameter, rules, is an
+        # "ingress_rules" structure parsed and validated by MU::Config.
+        #########################################################################
+        def setRules(rules, add_to_self: false, ingress: true, egress: false)
+          # XXX warn about attempt to set rules before we exist
+          return if rules.nil? or rules.size == 0 or !@cloud_id
+
+          # add_to_self means that this security is a "member" of its own rules
+          # (which is to say, objects that have this SG are allowed in my these
+          # rules)
+          if add_to_self
+            rules.each { |rule|
+              if rule['sgs'].nil? or !rule['sgs'].include?(@cloud_id)
+                new_rule = rule.clone
+                new_rule.delete('hosts')
+                rule['sgs'] = Array.new if rule['sgs'].nil?
+                rule['sgs'] << @cloud_id
               end
+            }
+          end
 
-              MU.log "Setting #{ingress ? "ingress" : "egress"} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
-              begin
-
-                if ingress
-                  if haverule
-                    begin
-                      MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
-                        group_id: @cloud_id,
-                        ip_permissions: [haverule]
-                      )
-                    rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
-                    end
+          ec2_rules = convertToEc2(rules)
+          return if ec2_rules.nil?
+
+          ext_permissions = MU.structToHash(cloud_desc(use_cache: false).ip_permissions)
+
+          purge_extraneous_rules(ec2_rules, ext_permissions)
+
+          ec2_rules.uniq!
+          ec2_rules.each { |rule|
+            haverule = nil
+            different = false
+            ext_permissions.each { |ext_rule|
+              if rule[:from_port] == ext_rule[:from_port] and
+                 rule[:to_port] == ext_rule[:to_port] and
+                 rule[:ip_protocol] == ext_rule[:ip_protocol]
+                haverule = ext_rule
+                ext_rule.keys.each { |k|
+                  if ext_rule[k].nil? or ext_rule[k] == []
+                    haverule.delete(k)
                   end
+                  different = true if rule[k] != ext_rule[k]
+                }
+                break
+              end
+            }
+            if haverule and !different
+              MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
+              next
+            end
+
+            MU.log "Setting #{ingress ? "ingress" : "egress"} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
+
+            MU.retrier([Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10, ignoreme: [Aws::EC2::Errors::InvalidPermissionDuplicate]) {
+              if ingress
+                if haverule
                   begin
-                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
+                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
                       group_id: @cloud_id,
-                      ip_permissions: [rule]
+                      ip_permissions: [haverule]
                     )
-                  rescue Aws::EC2::Errors::InvalidParameterCombination => e
-                    MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
-                    raise e
+                  rescue Aws::EC2::Errors::InvalidPermissionNotFound
                   end
                 end
-
-                if egress
-                  if haverule
-                    begin
-                      MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
-                        group_id: @cloud_id,
-                        ip_permissions: [haverule]
-                      )
-                    rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
-                    end
-                  end
-                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
+                begin
+                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
                     group_id: @cloud_id,
                     ip_permissions: [rule]
                   )
+                rescue Aws::EC2::Errors::InvalidParameterCombination => e
+                  MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
+                  raise e
                 end
+              end
 
-              rescue Aws::EC2::Errors::InvalidGroupNotFound => e
-                MU.log "#{@mu_name} (#{@cloud_id}) does not yet exist", MU::WARN
-                retries = retries + 1
-                if retries < 10
-                  sleep 10
-                  retry
-                else
-                  raise MuError, "#{@mu_name} does not exist", e.backtrace
+              if egress
+                if haverule
+                  begin
+                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
+                      group_id: @cloud_id,
+                      ip_permissions: [haverule]
+                    )
+                  rescue Aws::EC2::Errors::InvalidPermissionNotFound
+                  end
                 end
-              rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e
-                MU.log "Attempt to add duplicate rule to #{@mu_name}", MU::DEBUG, details: rule
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
+                  group_id: @cloud_id,
+                  ip_permissions: [rule]
+                )
               end
             }
-          end
+          }
 
         end
 
-        #########################################################################
-        # Convert our config languages description of firewall rules into Amazon's.
-        # This rule structure is as defined in MU::Config.
-        #########################################################################
+        #######################################################################
+        # Convert our config languages description of firewall rules into
+        # Amazon's. Our rule structure is as defined in MU::Config.
+        #######################################################################
         def convertToEc2(rules)
           ec2_rules = []
           if rules != nil
@@ -893,8 +860,11 @@ module MU
                 p_start = rule['port'].to_i
                 p_end = rule['port'].to_i
               elsif rule['proto'] != "icmp"
-                raise MuError, "Can't create a TCP or UDP security group rule without specifying ports: #{rule}"
+                MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule
+                p_start = "0"
+                p_end = "65535"
               end
+
               if rule['proto'] != "icmp"
                 if p_start.nil? or p_end.nil?
                   raise MuError, "Got nil ports out of rule #{rule}"
@@ -995,8 +965,8 @@ module MU
               ec2_rules << ec2_rule
             }
           end
-          ec2_rules.uniq!
-          return ec2_rules
+
+          ec2_rules.uniq
         end
 
       end #class