cloud-mu 3.1.3 → 3.3.0

data/modules/mu/{clouds → providers}/aws/loadbalancer.rb

@@ -163,7 +163,7 @@ module MU
           dnsthread = Thread.new {
             if !MU::Cloud::AWS.isGovCloud?
               MU.dupGlobals(parent_thread_id)
-              generic_mu_dns = MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: @mu_name, target: "#{lb.dns_name}.", cloudclass: MU::Cloud::LoadBalancer, sync_wait: @config['dns_sync_wait'])
+              generic_mu_dns = MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: @mu_name, target: "#{lb.dns_name}.", cloudclass: MU::Cloud::LoadBalancer, sync_wait: @config['dns_sync_wait'])
             end
           }
 
@@ -239,16 +239,35 @@ module MU
             end
           end
 
+          redirect_block = Proc.new { |r|
+            {
+              :protocol => r['protocol'],
+              :port => r['port'].to_s,
+              :host => r['host'],
+              :path => r['path'],
+              :query => r['query'],
+              :status_code => "HTTP_"+r['status_code'].to_s
+            }
+          }
+
           if !@config['classic']
             @config["listeners"].each { |l|
-              if !@targetgroups.has_key?(l['targetgroup'])
-                raise MuError, "Listener in #{@mu_name} configured for target group #{l['targetgroup']}, but I don't have data on a targetgroup by that name"
-              end
-              listen_descriptor = {
-                :default_actions => [{
+              action = if l['redirect']
+                {
+                  :type => "redirect",
+                  :redirect_config => redirect_block.call(l['redirect'])
+                }
+              else
+                if !@targetgroups.has_key?(l['targetgroup'])
+                  raise MuError, "Listener in #{@mu_name} configured for target group #{l['targetgroup']}, but I don't have data on a targetgroup by that name"
+                end
+                {
                   :target_group_arn => @targetgroups[l['targetgroup']].target_group_arn,
                   :type => "forward"
-                }], 
+                }
+              end
+              listen_descriptor = {
+                :default_actions => [ action ],
                 :load_balancer_arn => lb.load_balancer_arn,
                 :port => l['lb_port'], 
                 :protocol => l['lb_protocol']
@@ -276,10 +295,17 @@ module MU
                     :actions => []
                   }
                   rule['actions'].each { |a|
-                    rule_descriptor[:actions] << {
-                      :target_group_arn => @targetgroups[a['targetgroup']].target_group_arn,
-                      :type => a['action']
-                    }
+                    rule_descriptor[:actions] << if a['action'] == "forward"
+                      {
+                        :target_group_arn => @targetgroups[a['targetgroup']].target_group_arn,
+                        :type => a['action']
+                      }
+                    elsif a['action'] == "redirect"
+                      {
+                        :redirect_config => redirect_block.call(rule['redirect']),
+                        :type => a['action']
+                      }
+                    end
                   }
                   MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).create_rule(rule_descriptor)
                 }
@@ -288,7 +314,7 @@ module MU
           else
             @config["listeners"].each { |l|
               if l['ssl_certificate_id']
-                resp = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).set_load_balancer_policies_of_listener(
+                MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).set_load_balancer_policies_of_listener(
                   load_balancer_name: @cloud_id, 
                   load_balancer_port: l['lb_port'], 
                   policy_names: [
@@ -330,7 +356,7 @@ module MU
                 }
               )
             else
-              @targetgroups.each_pair { |tg_name, tg|
+              @targetgroups.values.each { |tg|
                 MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
                   target_group_arn: tg.target_group_arn,
                   attributes: [
@@ -400,7 +426,7 @@ module MU
                 timeout = 0
                 MU.log "Disabling connection draining on #{lb.dns_name}"
               end
-              @targetgroups.each_pair { |tg_name, tg|
+              @targetgroups.values.each { |tg|
                 MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
                   target_group_arn: tg.target_group_arn,
                   attributes: [
@@ -473,7 +499,7 @@ module MU
                 end
               end
             else
-              @targetgroups.each_pair { |tg_name, tg|
+              @targetgroups.values.each { |tg|
                 MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
                   target_group_arn: tg.target_group_arn,
                   attributes: [
@@ -536,7 +562,7 @@ module MU
               }
             end
             if !MU::Cloud::AWS.isGovCloud?
-              MU::Cloud::AWS::DNSZone.createRecordsFromConfig(@config['dns_records'], target: cloud_desc.dns_name)
+              MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: cloud_desc.dns_name)
             end
           end
 
@@ -553,15 +579,18 @@ module MU
           end
         end
 
+        @cloud_desc_cache = nil
         # Wrapper for cloud_desc method that deals with elb vs. elb2 resources.
-        def cloud_desc
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+          return nil if !@cloud_id
           if @config['classic']
-            resp = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
+            @cloud_desc_cache = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
               load_balancer_names: [@cloud_id]
             ).load_balancer_descriptions.first
-            return resp
+            return @cloud_desc_cache
           else
-            resp = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
+            @cloud_desc_cache = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
               names: [@cloud_id]
             ).load_balancers.first
             if @targetgroups.nil? and !@deploy.nil? and
@@ -573,7 +602,7 @@ module MU
               }
             end
 
-            return resp
+            return @cloud_desc_cache
           end
         end
 
@@ -643,8 +672,8 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          if (MU.deploy_id.nil? or MU.deploy_id.empty?) and (!flags or !flags["vpc_id"])
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          if (deploy_id.nil? or deploy_id.empty?) and (!flags or !flags["vpc_id"])
             raise MuError, "Can't touch ELBs without MU-ID or vpc_id flag"
           end
 
@@ -654,7 +683,7 @@ module MU
           # @param region [String]: The cloud provider region
           # @param ignoremaster [Boolean]: Whether to ignore the MU-MASTER-IP tag
           # @param classic [Boolean]: Whether to look for a classic ELB instead of an ALB (ELB2)
-          def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = false)
+          def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = false, deploy_id: MU.deploy_id)
             tags = []
             if classic
               tags = MU::Cloud::AWS.elb(credentials: credentials, region: region).describe_tags(
@@ -671,7 +700,7 @@ module MU
             if !tags.nil?
               tags.each { |tag|
                 saw_tags << tag.key
-                muid_match = true if tag.key == "MU-ID" and tag.value == MU.deploy_id
+                muid_match = true if tag.key == "MU-ID" and tag.value == deploy_id
                 mumaster_match = true if tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
               }
             end
@@ -692,20 +721,19 @@ module MU
               classic = false
             end
             begin
-              tags = []
               matched = false
               if flags and flags['vpc_id']
                 matched = true if lb.vpc_id == flags['vpc_id']
               else
                 if classic
-                  matched = self.checkForTagMatch(lb.load_balancer_name, region, ignoremaster, credentials, classic)
+                  matched = self.checkForTagMatch(lb.load_balancer_name, region, ignoremaster, credentials, classic, deploy_id: deploy_id)
                 else
-                  matched = self.checkForTagMatch(lb.load_balancer_arn, region, ignoremaster, credentials, classic)
+                  matched = self.checkForTagMatch(lb.load_balancer_arn, region, ignoremaster, credentials, classic, deploy_id: deploy_id)
                 end
               end
               if matched
                 if !MU::Cloud::AWS.isGovCloud?
-                  MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: lb.load_balancer_name, target: lb.dns_name, cloudclass: MU::Cloud::LoadBalancer, delete: true) if !noop
+                  MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: lb.load_balancer_name, target: lb.dns_name, cloudclass: MU::Cloud::LoadBalancer, delete: true) if !noop
                 end
                 if classic
                   MU.log "Removing Elastic Load Balancer #{lb.load_balancer_name}"
@@ -746,7 +774,7 @@ module MU
 
 
                   tgs.each { |tg|
-                    if self.checkForTagMatch(tg.target_group_arn, region, ignoremaster, credentials)
+                    if self.checkForTagMatch(tg.target_group_arn, region, ignoremaster, credentials, deploy_id: deploy_id)
                       MU.log "Removing Load Balancer Target Group #{tg.target_group_name}"
                       retries = 0
                       begin
@@ -773,9 +801,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "targetgroups" => {
@@ -792,35 +820,16 @@ module MU
                 }
               }
             },
-            "ingress_rules" => {
-              "items" => {
-                "properties" => {
-                  "sgs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  },
-                  "lbs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "AWS Load Balancers which will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  }
-                }
-              }
-            }
+            "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
           }
           [toplevel_required, schema]
         end
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::loadbalancers}, bare and unvalidated.
         # @param lb [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(lb, configurator)
+        def self.validateConfig(lb, _configurator)
           ok = true
 
           # XXX what about raw targetgroup ssl declarations?
@@ -829,8 +838,8 @@ module MU
                (!listener["ssl_certificate_id"].nil? and !listener["ssl_certificate_id"].empty?)
               if lb['cloud'] != "CloudFormation" # XXX or maybe do this anyway?
                 begin
-                  listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region'])
-                rescue MuError => e
+                  listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region']).first
+                rescue MuError
                   ok = false
                   next
                 end
@@ -922,6 +931,7 @@ module MU
           return matches
 
         end
+
       end
     end
   end

data/modules/mu/{clouds → providers}/aws/log.rb

@@ -202,11 +202,14 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::Log.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+          MU.log "Placeholder: AWS Log artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
+
           log_groups = self.find(credentials: credentials, region: region).values
           if !log_groups.empty?
             log_groups.each{ |lg|
-              if lg.log_group_name.match(MU.deploy_id)
+              if lg.log_group_name.match(deploy_id)
                 log_streams = MU::Cloud::AWS.cloudwatchlogs(credentials: credentials, region: region).describe_log_streams(log_group_name: lg.log_group_name).log_streams
                 if !log_streams.empty?
                   log_streams.each{ |ls|
@@ -227,13 +230,13 @@ module MU
             }
           end
 
-          unless noop
-            MU::Cloud::AWS.iam(credentials: credentials).list_roles.roles.each{ |role|
-              match_string = "#{MU.deploy_id}.*CloudTrail"
-              # Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud::AWS::Server.
-#              MU::Cloud::AWS::Server.removeIAMProfile(role.role_name) if role.role_name.match(match_string)
-            }
-          end
+#          unless noop
+#            MU::Cloud::AWS.iam(credentials: credentials).list_roles.roles.each{ |role|
+#              match_string = "#{deploy_id}.*CloudTrail"
+              # Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud.resourceClass("AWS", "Server").
+#              MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(match_string)
+#            }
+#          end
         end
 
         # Locate an existing log group.
@@ -264,7 +267,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
             "credentials" => @config['credentials'],
@@ -304,9 +307,9 @@ module MU
 
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "retention_period" => {
@@ -357,9 +360,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::logs}, bare and unvalidated.
         # @param log [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(log, configurator)
+        def self.validateConfig(log, _configurator)
           ok = true
 
           if log["filters"] && !log["filters"].empty?

data/modules/mu/{clouds → providers}/aws/msg_queue.rb

@@ -53,14 +53,14 @@ module MU
           new_attrs = genQueueAttrs
 
           changed = false
-          new_attrs.each_pair { |k, v|
+          new_attrs.each_pair { |k, _v|
             if !cur_attrs.has_key?(k) or cur_attrs[k] != new_attrs[k]
               changed = true
             end
           }
           if changed
             MU.log "Updating SQS queue #{@mu_name}", MU::NOTICE, details: new_attrs
-            resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).set_queue_attributes(
+            MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).set_queue_attributes(
               queue_url: @cloud_id,
               attributes: new_attrs
             )
@@ -74,10 +74,14 @@ module MU
           "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":sqs:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":"+@cloud_id
         end
 
+        @cloud_desc_cache = nil
         # Retrieve the AWS descriptor for this SQS queue. AWS doesn't exactly
         # provide one; if you want real information for SQS ask notify()
         # @return [Hash]: AWS doesn't return anything but the SQS URL, so supplement with attributes
-        def cloud_desc
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+          return nil if !@cloud_id
+
           if !@cloud_id
             resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).list_queues(
               queue_name_prefix: @mu_name
@@ -92,11 +96,12 @@ module MU
           end
 
           return nil if !@cloud_id
-          MU::Cloud::AWS::MsgQueue.find(
+          @cloud_desc_cache = MU::Cloud::AWS::MsgQueue.find(
             cloud_id: @cloud_id.dup,
             region: @config['region'],
             credentials: @config['credentials']
           )
+          @cloud_desc_cache
         end
 
         # Return the metadata for this MsgQueue rule
@@ -129,9 +134,12 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::MsgQueue.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+          MU.log "Placeholder: AWS MsgQueue artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
+
           resp = MU::Cloud::AWS.sqs(credentials: credentials, region: region).list_queues(
-            queue_name_prefix: MU.deploy_id
+            queue_name_prefix: deploy_id
           )
           if resp and resp.queue_urls
             threads = []
@@ -182,12 +190,20 @@ module MU
                 args[:cloud_id] = resp.queue_url if resp and resp.queue_url
               end
             end
-          rescue ::Aws::SQS::Errors::NonExistentQueue => e
+          rescue ::Aws::SQS::Errors::NonExistentQueue
           end
 
           # Go fetch its attributes
           fetch = if args[:cloud_id]
-            [args[:cloud_id]]
+            if args[:cloud_id] !~ /^https?:\/\//
+              [begin
+                MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).get_queue_url(queue_name: args[:cloud_id]).queue_url
+              rescue Aws::SQS::Errors::NonExistentQueue
+                return found
+              end]
+            else
+              [args[:cloud_id]]
+            end
           else
             resp = MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).list_queues
             resp.queue_urls
@@ -211,9 +227,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "max_msg_size" => {
@@ -320,16 +336,10 @@ module MU
               failq.delete("failqueue")
               ok = false if !configurator.insertKitten(failq, "msg_queues")
               queue['failqueue']['name'] = failq['name']
-              queue['dependencies'] << {
-                "name" => failq['name'],
-                "type" => "msg_queue"
-              }
+              MU::Config.addDependency(queue, failq["name"], "msg_queue")
             else
               if configurator.haveLitterMate?(queue['failqueue']['name'], "msg_queue")
-                queue['dependencies'] << {
-                  "name" => queue['failqueue']['name'],
-                  "type" => "msg_queue"
-                }
+                MU::Config.addDependency(queue, queue['failqueue']['name'], "msg_queue")
               else
                 failq = MU::Cloud::AWS::MsgQueue.find(cloud_id: queue['failqueue']['name'])
                 if !failq
@@ -382,7 +392,7 @@ module MU
             end
             begin
               MU::Cloud::AWS.kms(region: queue['region']).describe_key(key_id: queue['kms']['key_id'])
-            rescue Aws::KMS::Errors::NotFoundException => e
+            rescue Aws::KMS::Errors::NotFoundException
               MU.log "KMS key '#{queue['kms']['key_id']}' specified in Queue '#{queue['name']}' was not found.", MU::ERR, details: "Key IDs are of the form bf64a093-2c3d-46fa-0d4f-8232fa7ed53. Keys can be created at https://console.aws.amazon.com/iam/home#/encryptionKeys/#{queue['region']}"
               ok = false
             end