cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/clouds/aws/database.rb

@@ -19,38 +19,22 @@ module MU
     class AWS
       # A database as configured in {MU::Config::BasketofKittens::databases}
       class Database < MU::Cloud::Database
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :cloud_id
-        attr_reader :config
-        attr_reader :groomer    
-
-        @cloudformation_data = {}
-        attr_reader :cloudformation_data
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::databases}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          # @mu_name = mu_name ? mu_name : @deploy.getResourceName(@config["name"])
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @config["groomer"] = MU::Config.defaultGroomer unless @config["groomer"]
           @groomclass = MU::Groomer.loadGroomer(@config["groomer"])
 
-          if !mu_name.nil?
-            @mu_name = mu_name
-          else
-            @mu_name ||=
-              if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
-                @deploy.getResourceName(@config["name"], max_length: 15)
-              else
-                @deploy.getResourceName(@config["name"], max_length: 63)
-              end
+          @mu_name ||=
+            if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
+              @deploy.getResourceName(@config["name"], max_length: 15)
+            else
+              @deploy.getResourceName(@config["name"], max_length: 63)
+            end
 
-            @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
-          end
+          @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -189,8 +173,8 @@ module MU
         def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
           map = {}
           if cloud_id
-            db = MU::Cloud::AWS::Database.getDatabaseById(cloud_id, region: region, credentials: credentials)
-            map[cloud_id] = db if db
+            resp = MU::Cloud::AWS::Database.getDatabaseById(cloud_id, region: region, credentials: credentials)
+            map[cloud_id] = resp if resp
           end
 
           if tag_value
@@ -312,6 +296,7 @@ module MU
           
           if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
             config[:db_snapshot_identifier] = @config["snapshot_id"]
+            config[:db_cluster_identifier] = @config["cluster_identifier"] if @config["add_cluster_node"]
           end
 
           if @config["creation_style"] == "point_in_time"
@@ -384,11 +369,11 @@ module MU
             MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).wait_until(:db_instance_available, db_instance_identifier: @config['identifier']) do |waiter|
               # Does create_db_instance implement wait_until_available ?
               waiter.max_attempts = nil
-              waiter.before_attempt do |attempts|
-                MU.log "Waiting for RDS database #{@config['identifier']} to be ready..", MU::NOTICE if attempts % 10 == 0
+              waiter.before_attempt do |w_attempts|
+                MU.log "Waiting for RDS database #{@config['identifier']} to be ready..", MU::NOTICE if w_attempts % 10 == 0
               end
-              waiter.before_wait do |attempts, resp|
-                throw :success if resp.db_instances.first.db_instance_status == "available"
+              waiter.before_wait do |w_attempts, r|
+                throw :success if r.db_instances.first.db_instance_status == "available"
                 throw :failure if Time.now - wait_start_time > 3600
               end
             end
@@ -453,11 +438,11 @@ module MU
               MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).wait_until(:db_instance_available, db_instance_identifier: @config['identifier']) do |waiter|
                 # Does create_db_instance implement wait_until_available ?
                 waiter.max_attempts = nil
-                waiter.before_attempt do |attempts|
-                  MU.log "Waiting for RDS database #{@config['identifier'] } to be ready..", MU::NOTICE if attempts % 10 == 0
+                waiter.before_attempt do |w_attempts|
+                  MU.log "Waiting for RDS database #{@config['identifier'] } to be ready..", MU::NOTICE if w_attempts % 10 == 0
                 end
-                waiter.before_wait do |attempts, resp|
-                  throw :success if resp.db_instances.first.db_instance_status == "available"
+                waiter.before_wait do |w_attempts, r|
+                  throw :success if r.db_instances.first.db_instance_status == "available"
                   throw :failure if Time.now - wait_start_time > 2400
                 end
               end
@@ -532,6 +517,10 @@ module MU
             cluster_config_struct[:use_latest_restorable_time] = true if @config["restore_time"] == "latest"
           end
 
+          if @config['cloudwatch_logs']
+            cluster_config_struct[:enable_cloudwatch_logs_exports ] = @config['cloudwatch_logs']
+          end
+
           attempts = 0
           begin
             resp = 
@@ -655,8 +644,8 @@ module MU
               }
 
               @config['vpc'] = {
-                  "vpc_id" => vpc_id,
-                  "subnets" => mu_subnets
+                "vpc_id" => vpc_id,
+                "subnets" => mu_subnets
               }
               # Default VPC has only public subnets by default so setting publicly_accessible = true
               @config["publicly_accessible"] = true
@@ -798,7 +787,15 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          unless @config["create_cluster"]
+          if @config["create_cluster"]
+            @config['cluster_node_count'] ||= 1
+            if @config['cluster_mode'] == "serverless"
+              MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_current_db_cluster_capacity(
+                db_cluster_identifier: @cloud_id,
+                capacity: @config['cluster_node_count']
+              )
+            end
+          else
             database = MU::Cloud::AWS::Database.getDatabaseById(@config['identifier'], region: @config['region'], credentials: @config['credentials'])
 
             # Run SQL on deploy
@@ -1428,18 +1425,31 @@ module MU
             }
           }
 
+
           schema = {
             "db_parameter_group_parameters" => rds_parameters_primitive,
             "cluster_parameter_group_parameters" => rds_parameters_primitive,
+            "parameter_group_family" => {
+              "type" => "String",
+              "description" => "An RDS parameter group family. See also https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.html"
+            },
             "cluster_mode" => {
               "type" => "string",
               "description" => "The DB engine mode of the DB cluster",
               "enum" => ["provisioned", "serverless", "parallelquery", "global"],
               "default" => "provisioned"
             },
+            "cloudwatch_logs" => {
+              "type" => "array",
+              "default" => ["error"],
+              "items" => {
+                "type" => "string",
+                "enum" => ["error", "general", "audit", "slow_query"],
+              }
+            },
             "serverless_scaling" => {
               "type" => "object",
-              "descriptions" => "Scaling configuration for a +serverless+ Aurora cluster",
+              "description" => "Scaling configuration for a +serverless+ Aurora cluster",
               "default" => {
                 "auto_pause" => false,
                 "min_capacity" => 2,
@@ -1505,10 +1515,45 @@ module MU
         def self.validateConfig(db, configurator)
           ok = true
 
+          if db['creation_style'] == "existing_snapshot" and
+             !db['create_cluster'] and
+             db['identifier'] and db['identifier'].match(/:cluster-snapshot:/)
+            MU.log "Database #{db['name']}: Existing snapshot #{db['identifier']} looks like a cluster snapshot, but create_cluster is not set. Add 'create_cluster: true' if you're building an RDS cluster.", MU::ERR
+            ok = false
+          end
+
+          pgroup_families = []
+          engines = {}
+
+          marker = nil
+          begin
+            resp = MU::Cloud::AWS.rds(credentials: db['credentials'], region: db['region']).describe_db_engine_versions(marker: marker)
+            marker = resp.marker
+
+            if resp and resp.db_engine_versions
+              resp.db_engine_versions.each { |version|
+                engines[version.engine] ||= {
+                  "versions" => [],
+                  "families" => []
+                }
+                engines[version.engine]['versions'] << version.engine_version
+                engines[version.engine]['families'] << version.db_parameter_group_family
+
+              }
+              engines.keys.each { |engine|
+                engines[engine]["versions"].uniq!
+                engines[engine]["families"].uniq!
+              }
+
+            else
+              MU.log "Failed to get list of valid RDS engine versions in #{db['region']}, proceeding without proper validation", MU::WARN
+            end
+          end while !marker.nil?
+
           if db['create_cluster'] or db['engine'] == "aurora" or db["member_of_cluster"]
             case db['engine']
             when "mysql", "aurora", "aurora-mysql"
-              if db["engine_version"] == "5.6" or db["cluster_mode"] == "serverless"
+              if db["engine_version"].match(/^5\.6/) or db["cluster_mode"] == "serverless"
                 db["engine"] = "aurora"
               else
                 db["engine"] = "aurora-mysql"
@@ -1517,10 +1562,40 @@ module MU
               db["engine"] = "aurora-postgresql"
             else
               ok = false
-              MU.log "Requested a clustered database, but engine #{db['engine']} is not supported for clustering", MU::ERR
+              MU.log "Database #{db['name']}: Requested a clustered database, but engine #{db['engine']} is not supported for clustering", MU::ERR
             end
           end
 
+          if db['engine'].match(/^aurora/) and !db['create_cluster'] and !db['add_cluster_node']
+            MU.log "Database #{db['name']}: #{db['engine']} looks like a cluster engine, but create_cluster is not set. Add 'create_cluster: true' if you're building an RDS cluster.", MU::ERR
+            ok = false
+          end
+
+          if engines.size > 0 
+            if !engines[db['engine']]
+              MU.log "RDS engine #{db['engine']} is not supported in #{db['region']}", MU::ERR, details: engines.keys.sort
+              ok = false
+            else
+              if db["engine_version"] and
+                 engines[db['engine']]['versions'].size > 0 and
+                 !engines[db['engine']]['versions'].include?(db['engine_version']) and
+                 !engines[db['engine']]['versions'].grep(/^#{Regexp.quote(db["engine_version"])}.+/)
+                MU.log "RDS engine '#{db['engine']}' version '#{db['engine_version']}' is not supported in #{db['region']}", MU::ERR, details: { "Known-good versions:" => engines[db['engine']]['versions'].uniq.sort }
+                ok = false
+              end
+              if db["parameter_group_family"] and
+                 engines[db['engine']]['families'].size > 0 and
+                 !engines[db['engine']]['families'].include?(db['parameter_group_family'])
+                MU.log "RDS engine '#{db['engine']}' parameter group family '#{db['parameter_group_family']}' is not supported in #{db['region']}", MU::ERR, details: { "Valid parameter families:" => engines[db['engine']]['families'].uniq.sort }
+                ok = false
+              end
+            end
+          end
+
+          if db['parameter_group_family'] and pgroup_families.size > 0 and
+             !pgroup_families.include?(db['parameter_group_family'])
+          end
+
           db["license_model"] ||=
             if ["postgres", "postgresql", "aurora-postgresql"].include?(db["engine"])
               "postgresql-license"
@@ -1605,7 +1680,7 @@ module MU
           end
 
           if db["vpc"]
-            if db["vpc"]["subnet_pref"] == "all_public" and !db['publicly_accessible']
+            if db["vpc"]["subnet_pref"] == "all_public" and !db['publicly_accessible'] and (db["vpc"]['subnets'].nil? or db["vpc"]['subnets'].empty?)
               MU.log "Setting publicly_accessible to true on database '#{db['name']}', since deploying into public subnets.", MU::WARN
               db['publicly_accessible'] = true
             elsif db["vpc"]["subnet_pref"] == "all_private" and db['publicly_accessible']

data/modules/mu/clouds/aws/dnszone.rb

@@ -19,22 +19,11 @@ module MU
       # A DNS Zone as configured in {MU::Config::BasketofKittens::dnszones}
       class DNSZone < MU::Cloud::DNSZone
 
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :cloud_id
-        attr_reader :config
-
-        @cloudformation_data = {}
-        attr_reader :cloudformation_data
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::dnszones}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          unless @mu_name
-            @mu_name = mu_name ? mu_name : @deploy.getResourceName(@config["name"])
-          end
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
+          @mu_name ||= @deploy.getResourceName(@config["name"])
 
           MU.setVar("curRegion", @config['region']) if !@config['region'].nil?
         end
@@ -399,8 +388,8 @@ module MU
             if !alias_zone.nil?
               target_zone = "/hostedzone/"+alias_zone if !alias_zone.match(/^\/hostedzone\//)
             else
-              MU::Cloud::AWS.listRegions.each { |region|
-                MU::Cloud::AWS.elb(region: region).describe_load_balancers.load_balancer_descriptions.each { |elb|
+              MU::Cloud::AWS.listRegions.each { |r|
+                MU::Cloud::AWS.elb(region: r).describe_load_balancers.load_balancer_descriptions.each { |elb|
                   elb_dns = elb.dns_name.downcase
                   elb_dns.chomp!(".")
                   if target_name == elb_dns

data/modules/mu/clouds/aws/endpoint.rb

@@ -3,21 +3,11 @@ module MU
     class AWS
       # An API as configured in {MU::Config::BasketofKittens::endpoints}
       class Endpoint < MU::Cloud::Endpoint
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :cloud_id
-
-        @cloudformation_data = {}
-        attr_reader :cloudformation_data
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::endpoints}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @mu_name ||= @deploy.getResourceName(@config["name"])
         end
 

data/modules/mu/clouds/aws/firewall_rule.rb

@@ -18,30 +18,19 @@ module MU
     class AWS
       # A firewall ruleset as configured in {MU::Config::BasketofKittens::firewall_rules}
       class FirewallRule < MU::Cloud::FirewallRule
+        require "mu/clouds/aws/vpc"
 
-        @deploy = nil
-        @config = nil
         @admin_sgs = Hash.new
         @admin_sg_semaphore = Mutex.new
 
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :cloud_id
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::firewall_rules}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          if !mu_name.nil?
-            @mu_name = mu_name
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
+          if !@vpc.nil?
+            @mu_name ||= @deploy.getResourceName(@config['name'], need_unique_string: true)
           else
-            if !@vpc.nil?
-              @mu_name = @deploy.getResourceName(@config['name'], need_unique_string: true)
-            else
-              @mu_name = @deploy.getResourceName(@config['name'])
-            end
+            @mu_name ||= @deploy.getResourceName(@config['name'])
           end
 
         end
@@ -86,7 +75,7 @@ module MU
             retry
           end
 
-          MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
+          MU::Cloud::AWS.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
           MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
 
           if @config['optional_tags']
@@ -170,7 +159,6 @@ module MU
           else
             rule["port_range"] = port_range
           end
-          rule["description"] = comment if comment
           ec2_rule = convertToEc2([rule])
 
           begin
@@ -212,32 +200,27 @@ module MU
         end
 
         # Locate an existing security group or groups and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching FirewallRules
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
+        def self.find(**args)
 
-          if !cloud_id.nil? and !cloud_id.empty?
+          if !args[:cloud_id].nil? and !args[:cloud_id].empty?
             begin
-              resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(group_ids: [cloud_id])
-              return {cloud_id => resp.data.security_groups.first}
+              resp = MU::Cloud::AWS.ec2(region: args[:region], credentials: args[:credentials]).describe_security_groups(group_ids: [args[:cloud_id]])
+              return {args[:cloud_id] => resp.data.security_groups.first}
             rescue ArgumentError => e
-              MU.log "Attempting to load #{cloud_id}: #{e.inspect}", MU::WARN, details: caller
+              MU.log "Attempting to load #{args[:cloud_id]}: #{e.inspect}", MU::WARN, details: caller
               return {}
             rescue Aws::EC2::Errors::InvalidGroupNotFound => e
-              MU.log "Attempting to load #{cloud_id}: #{e.inspect}", MU::DEBUG, details: caller
+              MU.log "Attempting to load #{args[:cloud_id]}: #{e.inspect}", MU::DEBUG, details: caller
               return {}
             end
           end
 
           map = {}
-          if !tag_key.nil? and !tag_value.nil?
-            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
+          if !args[:tag_key].nil? and !args[:tag_value].nil?
+            resp = MU::Cloud::AWS.ec2(region: args[:region], credentials: args[:credentials]).describe_security_groups(
                 filters: [
-                    {name: "tag:#{tag_key}", values: [tag_value]}
+                    {name: "tag:#{args[:tag_key]}", values: [args[:tag_value]]}
                 ]
             )
             if !resp.nil?
@@ -269,19 +252,26 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          tagfilters = [
+          filters = nil
+          if flags and flags["vpc_id"]
+            filters = [
+              {name: "vpc-id", values: [flags["vpc_id"]]}
+            ]
+          else
+            filters = [
               {name: "tag:MU-ID", values: [MU.deploy_id]}
-          ]
-          if !ignoremaster
-            tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
+            ]
+            if !ignoremaster
+              filters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
+            end
           end
 
           # Some services create sneaky rogue ENIs which then block removal of
           # associated security groups. Find them and fry them.
-          MU::Cloud::AWS::VPC.purge_interfaces(noop, tagfilters, region: region, credentials: credentials)
+          MU::Cloud::AWS::VPC.purge_interfaces(noop, filters, region: region, credentials: credentials)
 
           resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
-              filters: tagfilters
+            filters: filters
           )
 
           resp.data.security_groups.each { |sg|
@@ -361,11 +351,14 @@ module MU
           }
 
           resp.data.security_groups.each { |sg|
+            next if sg.group_name == "default"
             MU.log "Removing EC2 Security Group #{sg.group_name}"
 
             retries = 0
             begin
               MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id) if !noop
+            rescue Aws::EC2::Errors::CannotDelete => e
+              MU.log e.message, MU::WARN
             rescue Aws::EC2::Errors::InvalidGroupNotFound
               MU.log "EC2 Security Group #{sg.group_name} disappeared before I could delete it!", MU::WARN
             rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
@@ -488,28 +481,109 @@ module MU
           end
 
           ec2_rules = convertToEc2(rules)
+          ext_permissions = MU.structToHash(cloud_desc.ip_permissions)
+
+          # Purge any old rules that we're sure we created (check the comment)
+          # but which are no longer configured.
+          ext_permissions.each { |ext_rule|
+            haverule = false
+            ec2_rules.each { |rule|
+              if rule[:from_port] == ext_rule[:from_port] and
+                 rule[:to_port] == ext_rule[:to_port] and
+                 rule[:ip_protocol] == ext_rule[:ip_protocol]
+                haverule = true
+                break
+              end
+            }
+            next if haverule
+
+            mu_comments = false
+            (ext_rule[:user_id_group_pairs] + ext_rule[:ip_ranges]).each { |entry|
+              if entry[:description] == "Added by Mu"
+                mu_comments = true
+              else
+                mu_comments = false
+                break
+              end
+            }
+
+            if mu_comments
+              ext_rule.keys.each { |k|
+                if ext_rule[k].nil? or ext_rule[k] == []
+                  ext_rule.delete(k)
+                end
+              }
+              MU.log "Removing unconfigured rule in #{@mu_name}", MU::WARN, details: ext_rule
+              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
+                group_id: @cloud_id,
+                ip_permissions: [ext_rule]
+              )
+            end
+
+          }
 
           # Creating an empty security group is ok, so don't freak out if we get
           # a null rule list.
           if !ec2_rules.nil?
             ec2_rules.uniq!
-            MU.log "Setting rules in Security Group #{@mu_name} (#{@cloud_id})", details: ec2_rules
             retries = 0
-            if rules != nil
-              MU.log "Rules for EC2 Security Group #{@mu_name} (#{@cloud_id}): #{ec2_rules}", MU::DEBUG
+            ec2_rules.each { |rule|
+              haverule = nil
+              different = false
+              ext_permissions.each { |ext_rule|
+                if rule[:from_port] == ext_rule[:from_port] and
+                   rule[:to_port] == ext_rule[:to_port] and
+                   rule[:ip_protocol] == ext_rule[:ip_protocol]
+                  haverule = ext_rule
+                  ext_rule.keys.each { |k|
+                    if ext_rule[k].nil? or ext_rule[k] == []
+                      haverule.delete(k)
+                    end
+                    different = true if rule[k] != ext_rule[k]
+                  }
+                  break
+                end
+              }
+              if haverule and !different
+                MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
+                next
+              end
+
+              MU.log "Setting #{ingress ? "ingress" : "egress"} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
               begin
+
                 if ingress
+                  if haverule
+                    begin
+                      MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
+                        group_id: @cloud_id,
+                        ip_permissions: [haverule]
+                      )
+                    rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
+                    end
+                  end
                   MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
-                      group_id: @cloud_id,
-                      ip_permissions: ec2_rules
+                    group_id: @cloud_id,
+                    ip_permissions: [rule]
                   )
                 end
+
                 if egress
+                  if haverule
+                    begin
+                      MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
+                        group_id: @cloud_id,
+                        ip_permissions: [haverule]
+                      )
+                    rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
+                    end
+                  end
                   MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
-                      group_id: @cloud_id,
-                      ip_permissions: ec2_rules
+                    group_id: @cloud_id,
+                    ip_permissions: [rule]
                   )
                 end
+
               rescue Aws::EC2::Errors::InvalidGroupNotFound => e
                 MU.log "#{@mu_name} (#{@cloud_id}) does not yet exist", MU::WARN
                 retries = retries + 1
@@ -520,9 +594,9 @@ module MU
                   raise MuError, "#{@mu_name} does not exist", e.backtrace
                 end
               rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e
-                MU.log "Attempt to add duplicate rule to #{@mu_name}", MU::DEBUG, details: ec2_rules
+                MU.log "Attempt to add duplicate rule to #{@mu_name}", MU::DEBUG, details: rule
               end
-            end
+            }
           end
 
         end
@@ -538,6 +612,8 @@ module MU
 
             rules.each { |rule|
               ec2_rule = {}
+              rule["comment"] ||= "Added by Mu"
+
 
               rule['proto'] ||= "tcp"
               ec2_rule[:ip_protocol] = rule['proto']
@@ -564,9 +640,9 @@ module MU
               end
 
               if (!defined? rule['hosts'] or !rule['hosts'].is_a?(Array)) and
-                  (!defined? rule['sgs'] or !rule['sgs'].is_a?(Array)) and
-                  (!defined? rule['lbs'] or !rule['lbs'].is_a?(Array))
-                raise MuError, "One of 'hosts', 'sgs', or 'lbs' in rules provided to createEc2SG must be an array."
+                 (!defined? rule['sgs'] or !rule['sgs'].is_a?(Array)) and
+                 (!defined? rule['lbs'] or !rule['lbs'].is_a?(Array))
+                rule['hosts'] = ["0.0.0.0/0"]
               end
               ec2_rule[:ip_ranges] = []
               ec2_rule[:user_id_group_pairs] = []
@@ -576,11 +652,7 @@ module MU
                 rule['hosts'].each { |cidr|
                   next if cidr.nil? # XXX where is that coming from?
                   cidr = cidr + "/32" if cidr.match(/^\d+\.\d+\.\d+\.\d+$/)
-                  if rule['description']
-                    ec2_rule[:ip_ranges] << {cidr_ip: cidr, description: rule['description']}
-                  else
-                    ec2_rule[:ip_ranges] << {cidr_ip: cidr}
-                  end
+                  ec2_rule[:ip_ranges] << {cidr_ip: cidr, description: rule['comment']}
                 }
               end
 
@@ -627,25 +699,25 @@ module MU
                 rule['sgs'].uniq!
                 rule['sgs'].each { |sg_name|
                   dependencies # Make sure our cache is fresh
-                  if sg_name == @config['name']
-                    sg = self
+                  sg = @deploy.findLitterMate(type: "firewall_rule", name: sg_name) if @deploy
+                  sg ||= if sg_name == @config['name']
+                    self
                   elsif @dependencies.has_key?("firewall_rule") and
                       @dependencies["firewall_rule"].has_key?(sg_name)
-                    sg = @dependencies["firewall_rule"][sg_name]
-                  else
-                    if sg_name.match(/^sg-/)
-                      found_sgs = MU::MommaCat.findStray("AWS", "firewall_rule", cloud_id: sg_name, region: @config['region'], calling_deploy: @deploy, dummy_ok: true)
-                    else
-                      found_sgs = MU::MommaCat.findStray("AWS", "firewall_rule", name: sg_name, region: @config['region'], deploy_id: MU.deploy_id, calling_deploy: @deploy)
-                    end
-                    if found_sgs.nil? or found_sgs.size == 0
-                      raise MuError, "Attempted to reference non-existent Security Group #{sg_name} while building #{@mu_name}"
-                    end
-                    sg = found_sgs.first
+                    @dependencies["firewall_rule"][sg_name]
+                  elsif sg_name.match(/^sg-/)
+                    found_sgs = MU::MommaCat.findStray("AWS", "firewall_rule", cloud_id: sg_name, region: @config['region'], calling_deploy: @deploy, dummy_ok: true)
+                    found_sgs.first if found_sgs
                   end
+
+                  if sg.nil?
+                    raise MuError, "FirewallRule #{@config['name']} referenced security group '#{sg_name}' in a rule, but I can't find it anywhere!"
+                  end
+
                   ec2_rule[:user_id_group_pairs] << {
                     user_id: MU.account_number,
-                    group_id: sg.cloud_id
+                    group_id: sg.cloud_id,
+                    description: rule['comment']
                   }
                 }
               end