cloud-mu 3.5.0 → 3.6.3

data/modules/mu/providers/aws/firewall_rule.rb

@@ -107,6 +107,7 @@ module MU
         # Called by {MU::Deploy#createResources}
         def groom
           if !@config['rules'].nil? and @config['rules'].size > 0
+
             egress = false
             egress = true if !@vpc.nil?
             # XXX the egress logic here is a crude hack, this really needs to be
@@ -172,11 +173,13 @@ module MU
 
           begin
             if egress
+MU.log ".authorize_security_group_egress 2", MU::NOTICE, details: ec2_rule
               MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_egress(
                 group_id: @cloud_id,
                 ip_permissions: ec2_rule
               )
             else
+MU.log ".authorize_security_group_ingress 2", MU::NOTICE, details: ec2_rule
               MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_ingress(
                 group_id: @cloud_id,
                 ip_permissions: ec2_rule
@@ -188,11 +191,13 @@ module MU
             # existing rules
             if comment
               if egress
+MU.log ".update_security_group_rule_descriptions_engress", MU::NOTICE, details: ec2_rule
                 MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).update_security_group_rule_descriptions_egress(
                   group_id: @cloud_id,
                   ip_permissions: ec2_rule
                 )
               else
+MU.log ".update_security_group_rule_descriptions_ingress", MU::NOTICE, details: ec2_rule
                 MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).update_security_group_rule_descriptions_ingress(
                   group_id: @cloud_id,
                   ip_permissions: ec2_rule
@@ -702,15 +707,14 @@ module MU
 
         private
 
-        def purge_extraneous_rules(ec2_rules, ext_permissions)
+        def purge_extraneous_rules(ec2_rules, ext_permissions, egress: false)
           # Purge any old rules that we're sure we created (check the comment)
-          # but which are no longer configured.
+          # but which are no longer configured. Also purge rules that overlap
+          # with new rules.
           ext_permissions.each { |ext_rule|
             haverule = false
             ec2_rules.each { |rule|
-              if rule[:from_port] == ext_rule[:from_port] and
-                 rule[:to_port] == ext_rule[:to_port] and
-                 rule[:ip_protocol] == ext_rule[:ip_protocol]
+              if rules_match(rule, ext_rule)
                 haverule = true
                 break
               end
@@ -719,7 +723,7 @@ module MU
 
             mu_comments = false
             (ext_rule[:user_id_group_pairs] + ext_rule[:ip_ranges]).each { |entry|
-              if entry[:description] == "Added by Mu"
+              if entry[:description] and !entry[:description].empty?
                 mu_comments = true
               else
                 mu_comments = false
@@ -733,22 +737,60 @@ module MU
                   ext_rule.delete(k)
                 end
               }
-              MU.log "Removing unconfigured rule in #{@mu_name}", MU::WARN, details: ext_rule
-              MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_ingress(
-                group_id: @cloud_id,
-                ip_permissions: [ext_rule]
-              )
+              MU.log "Removing changed #{egress ? "egress" : "ingress"} rule in #{@mu_name}", MU::WARN, details: ext_rule
+              begin
+                if !egress
+                  MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_ingress(
+                    group_id: @cloud_id,
+                    ip_permissions: [ext_rule]
+                  )
+                else
+                  MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_egress(
+                    group_id: @cloud_id,
+                    ip_permissions: [ext_rule]
+                  )
+                end
+              rescue Aws::EC2::Errors::InvalidPermissionNotFound
+              end
             end
           }
         end
 
-        #########################################################################
+        def rules_match(rule_a, rule_b)
+          (
+           rule_a[:from_port] == rule_b[:from_port] and
+           rule_a[:to_port] == rule_b[:to_port] and
+           rule_a[:ip_protocol] == rule_b[:ip_protocol] and
+           (
+            (
+             (rule_a[:ip_ranges].nil? or rule_a[:ip_ranges].empty?) and
+             (rule_b[:ip_ranges].nil? or rule_b[:ip_ranges].empty?)
+            ) or
+            (
+             !rule_a[:ip_ranges].nil? and !rule_b[:ip_ranges].nil? and
+             rule_a[:ip_ranges].sort == rule_b[:ip_ranges].sort
+            )
+           ) and
+           (
+            (
+             (rule_a[:user_id_group_pairs].nil? or rule_a[:user_id_group_pairs].empty?) and 
+             (rule_b[:user_id_group_pairs].nil? or rule_b[:user_id_group_pairs].empty?) 
+            ) or
+            (
+             !rule_a[:user_id_group_pairs].nil? and !rule_b[:user_id_group_pairs].nil? and
+             rule_a[:user_id_group_pairs].sort == rule_b[:user_id_group_pairs].sort
+            )
+           )
+          )
+        end
+
+        ########################################################################
         # Manufacture an EC2 security group. The second parameter, rules, is an
         # "ingress_rules" structure parsed and validated by MU::Config.
-        #########################################################################
+        ########################################################################
         def setRules(rules, add_to_self: false, ingress: true, egress: false)
           # XXX warn about attempt to set rules before we exist
-          return if rules.nil? or rules.size == 0 or !@cloud_id
+          return if rules.nil? or rules.empty? or !@cloud_id
 
           # add_to_self means that this security is a "member" of its own rules
           # (which is to say, objects that have this SG are allowed in my these
@@ -766,73 +808,94 @@ module MU
 
           ec2_rules = convertToEc2(rules)
           return if ec2_rules.nil?
+          ec2_rules.uniq!
 
-          ext_permissions = MU.structToHash(cloud_desc(use_cache: false).ip_permissions)
+          fresh_desc = cloud_desc(use_cache: false)
+          directions = []
+          directions << :ingress if ingress
+          directions << :egress if egress
 
-          purge_extraneous_rules(ec2_rules, ext_permissions)
+          directions.each { |dir|
+            ext_permissions = MU.structToHash(dir == :ingress ? fresh_desc.ip_permissions : fresh_desc.ip_permissions_egress)
+            purge_extraneous_rules(ec2_rules, ext_permissions, egress: (dir == :egress))
 
-          ec2_rules.uniq!
-          ec2_rules.each { |rule|
-            haverule = nil
-            different = false
-            ext_permissions.each { |ext_rule|
-              if rule[:from_port] == ext_rule[:from_port] and
-                 rule[:to_port] == ext_rule[:to_port] and
-                 rule[:ip_protocol] == ext_rule[:ip_protocol]
-                haverule = ext_rule
-                ext_rule.keys.each { |k|
-                  if ext_rule[k].nil? or ext_rule[k] == []
-                    haverule.delete(k)
-                  end
-                  different = true if rule[k] != ext_rule[k]
-                }
-                break
+            ec2_rules.each { |rule|
+              haverule = nil
+              different = false
+              ext_permissions.each { |ext_rule|
+                if rules_match(rule, ext_rule)
+                  haverule = ext_rule
+                  ext_rule.keys.each { |k|
+                    if ext_rule[k].nil? or ext_rule[k] == []
+                      haverule.delete(k)
+                    end
+                    different = true if rule[k] != ext_rule[k]
+                  }
+                  break
+                end
+              }
+              if haverule and !different
+                MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
+                next
               end
-            }
-            if haverule and !different
-              MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
-              next
-            end
 
-            MU.log "Setting #{ingress ? "ingress" : "egress"} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
+              MU.log "Setting #{dir.to_s} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
 
-            MU.retrier([Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10, ignoreme: [Aws::EC2::Errors::InvalidPermissionDuplicate]) {
-              if ingress
+              MU.retrier([Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10) {
                 if haverule
                   begin
-                    MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_ingress(
-                      group_id: @cloud_id,
-                      ip_permissions: [haverule]
-                    )
+                    params = {
+                      :group_id => @cloud_id,
+                      :ip_permissions => [haverule]
+                    }
+                    MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).send("revoke_security_group_#{dir.to_s}".to_sym, params)
                   rescue Aws::EC2::Errors::InvalidPermissionNotFound
                   end
                 end
+                remove_me = nil
+                removals = 0
                 begin
-                  MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_ingress(
-                    group_id: @cloud_id,
-                    ip_permissions: [rule]
-                  )
+                  if remove_me
+                    removals += 1
+                    MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).send("revoke_security_group_#{dir.to_s}".to_sym, remove_me)
+                  end
+                  params = {
+                    :group_id => @cloud_id,
+                    :ip_permissions => [rule]
+                  }
+                  MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).send("authorize_security_group_#{dir.to_s}".to_sym, params)
+                rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e
+
+                  if e.message =~ /"peer: (\d+\.\d+\.\d+\.\d+\/\d+), (TCP|UDP|ICMP|), from port: (\d+), to port: (\d+), ALLOW"/ and removals < 50
+                    MU.log "FirewallRule #{@mu_name} attempted to add a duplicate rule: #{e.message}, will attempt to remove", MU::NOTICE
+                    remove_me = {
+                      :group_id => @cloud_id,
+                      :ip_permissions => [
+                        {
+                          :from_port => Regexp.last_match[3].to_i,
+                          :to_port => Regexp.last_match[3].to_i,
+                          :ip_protocol => Regexp.last_match[2].downcase,
+                          :ip_ranges => [
+                            :cidr_ip => Regexp.last_match[1]
+                          ]
+                        }
+                      ]
+                    }
+                    retry
+                  else
+                    MU.log "FirewallRule #{@mu_name} attempted to add a duplicate rule: #{e.message}", MU::WARN, details: [cloud_desc, params]
+                  end
+                rescue Aws::EC2::Errors::InvalidParameterValue => e
+                  if e.message =~ /The same permission must not appear multiple times/
+                    MU.log "FirewallRule #{@mu_name} attempted to add a duplicate rule: #{e.message}", MU::WARN, details: [cloud_desc, params]
+                  else
+                    raise e
+                  end
                 rescue Aws::EC2::Errors::InvalidParameterCombination => e
                   MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
                   raise e
                 end
-              end
-
-              if egress
-                if haverule
-                  begin
-                    MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_egress(
-                      group_id: @cloud_id,
-                      ip_permissions: [haverule]
-                    )
-                  rescue Aws::EC2::Errors::InvalidPermissionNotFound
-                  end
-                end
-                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_egress(
-                  group_id: @cloud_id,
-                  ip_permissions: [rule]
-                )
-              end
+              }
             }
           }
 
@@ -843,131 +906,150 @@ module MU
         # Amazon's. Our rule structure is as defined in MU::Config.
         #######################################################################
         def convertToEc2(rules)
+          return [] if rules.nil?
           ec2_rules = []
-          if rules != nil
-            rules.uniq!
 
-            rules.each { |rule|
-              ec2_rule = {}
-              rule["comment"] ||= "Added by Mu"
-
-
-              rule['proto'] ||= "tcp"
-              ec2_rule[:ip_protocol] = rule['proto']
-
-              p_start = nil
-              p_end = nil
-              if rule['port_range']
-                p_start, p_end = rule['port_range'].to_s.split(/\s*-\s*/)
-              elsif rule['port']
-                p_start = rule['port'].to_i
-                p_end = rule['port'].to_i
-              elsif rule['proto'] != "icmp"
-                MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule
-                p_start = "0"
-                p_end = "65535"
-              end
+          rules.uniq!
 
-              if rule['proto'] != "icmp"
-                if p_start.nil? or p_end.nil?
-                  raise MuError, "Got nil ports out of rule #{rule}"
-                end
-                ec2_rule[:from_port] = p_start.to_i
-                ec2_rule[:to_port] = p_end.to_i
-              else
-                ec2_rule[:from_port] = -1
-                ec2_rule[:to_port] = -1
-              end
+          rules.each { |rule|
+            ec2_rule = {}
+            rule["comment"] ||= "Added by Mu"
 
-              if (!defined? rule['hosts'] or !rule['hosts'].is_a?(Array)) and
-                 (!defined? rule['firewall_rules'] or !rule['firewall_rules'].is_a?(Array)) and
-                 (!defined? rule['loadbalancers'] or !rule['loadbalancers'].is_a?(Array))
-                rule['hosts'] = ["0.0.0.0/0"]
-              end
-              ec2_rule[:ip_ranges] = []
-              ec2_rule[:user_id_group_pairs] = []
-
-              if !rule['hosts'].nil?
-                rule['hosts'].uniq!
-                rule['hosts'].each { |cidr|
-                  next if cidr.nil? # XXX where is that coming from?
-                  cidr = cidr + "/32" if cidr.match(/^\d+\.\d+\.\d+\.\d+$/)
-                  ec2_rule[:ip_ranges] << {cidr_ip: cidr, description: rule['comment']}
-                }
-              end
 
-              if !rule['loadbalancers'].nil?
-                rule['loadbalancers'].uniq!
-                rule['loadbalancers'].each { |lb|
-                  lb_ref = MU::Config::Ref.get(lb)
+            rule['proto'] ||= "tcp"
+            ec2_rule[:ip_protocol] = rule['proto']
 
-                  if !lb_ref.kitten or !lb_ref.kitten.cloud_desc
-                    MU.log "Security Group #{@mu_name} failed to get cloud descriptor for referenced load balancer", MU::ERR, details: lb_ref
-                    next
-                  end
+            p_start = nil
+            p_end = nil
+            if rule['port_range']
+              p_start, p_end = rule['port_range'].to_s.split(/\s*-\s*/)
+            elsif rule['port']
+              p_start = rule['port'].to_i
+              p_end = rule['port'].to_i
+            elsif rule['proto'] != "icmp"
+              MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule
+              p_start = "0"
+              p_end = "65535"
+            end
 
-                  lb_ref.kitten.cloud_desc.security_groups.each { |lb_sg|
-                    # XXX this probably has to infer things like region,
-                    # credentials, etc from the load balancer ref
-                    lb_sg_desc = MU::Cloud::AWS::FirewallRule.find(cloud_id: lb_sg)
-                    owner_id = if lb_sg_desc and lb_sg_desc.size == 1
-                      lb_sg_desc.values.first.owner_id
-                    else
-                      MU::Cloud::AWS.credToAcct(@credentials)
-                    end
-                    ec2_rule[:user_id_group_pairs] << {
-                      user_id: owner_id,
-                      group_id: lb_sg,
-                      description: rule['comment']
-                    }
-                  }
-                }
+            if rule['proto'] != "icmp"
+              if p_start.nil? or p_end.nil?
+                raise MuError, "Got nil ports out of rule #{rule}"
               end
+              ec2_rule[:from_port] = p_start.to_i
+              ec2_rule[:to_port] = p_end.to_i
+            else
+              ec2_rule[:from_port] = -1
+              ec2_rule[:to_port] = -1
+            end
 
-              if !rule['firewall_rules'].nil?
-                rule['firewall_rules'].uniq!
-                rule['firewall_rules'].each { |sg|
-                  sg_ref = MU::Config::Ref.get(sg)
+            if (!defined? rule['hosts'] or !rule['hosts'].is_a?(Array)) and
+               (!defined? rule['firewall_rules'] or !rule['firewall_rules'].is_a?(Array)) and
+               (!defined? rule['loadbalancers'] or !rule['loadbalancers'].is_a?(Array))
+              rule['hosts'] = ["0.0.0.0/0"]
+            end
+            ec2_rule[:ip_ranges] = []
+            ec2_rule[:user_id_group_pairs] = []
+
+            if !rule['hosts'].nil?
+              rule['hosts'].uniq!
+              rule['hosts'].each { |cidr|
+                next if cidr.nil? # XXX where is that coming from?
+                cidr = cidr + "/32" if cidr.match(/^\d+\.\d+\.\d+\.\d+$/)
+                ec2_rule[:ip_ranges] << {cidr_ip: cidr, description: rule['comment']}
+              }
+            end
 
-                  if !sg_ref.kitten or !sg_ref.kitten.cloud_desc
-                    MU.log "Security Group #{@mu_name} failed to get cloud descriptor for referenced Security Group", MU::ERR, details: sg_ref
-                    next
-                  end
+            if !rule['loadbalancers'].nil?
+              rule['loadbalancers'].uniq!
+              rule['loadbalancers'].each { |lb|
+                lb_ref = MU::Config::Ref.get(lb)
 
+                if !lb_ref.kitten or !lb_ref.kitten.cloud_desc
+                  MU.log "Security Group #{@mu_name} failed to get cloud descriptor for referenced load balancer", MU::ERR, details: lb_ref
+                  next
+                end
+
+                lb_ref.kitten.cloud_desc.security_groups.each { |lb_sg|
+                  # XXX this probably has to infer things like region,
+                  # credentials, etc from the load balancer ref
+                  lb_sg_desc = MU::Cloud::AWS::FirewallRule.find(cloud_id: lb_sg)
+                  owner_id = if lb_sg_desc and lb_sg_desc.size == 1
+                    lb_sg_desc.values.first.owner_id
+                  else
+                    MU::Cloud::AWS.credToAcct(@credentials)
+                  end
                   ec2_rule[:user_id_group_pairs] << {
-                    user_id: sg_ref.kitten.cloud_desc.owner_id,
-                    group_id: sg_ref.cloud_id,
+                    user_id: owner_id,
+                    group_id: lb_sg,
                     description: rule['comment']
                   }
+                }
+              }
+            end
+
+            if !rule['firewall_rules'].nil?
+              rule['firewall_rules'].uniq!
+              rule['firewall_rules'].each { |sg|
+                sg_ref = MU::Config::Ref.get(sg)
+
+                if !sg_ref.kitten or !sg_ref.kitten.cloud_desc
+                  MU.log "Security Group #{@mu_name} failed to get cloud descriptor for referenced Security Group", MU::ERR, details: sg_ref
+                  next
+                end
 
+                ec2_rule[:user_id_group_pairs] << {
+                  user_id: sg_ref.kitten.cloud_desc.owner_id,
+                  group_id: sg_ref.cloud_id,
+                  description: rule['comment']
                 }
-              end
 
-              ec2_rule[:user_id_group_pairs].uniq!
-              ec2_rule[:ip_ranges].uniq!
-              ec2_rule.delete(:ip_ranges) if ec2_rule[:ip_ranges].empty?
-              ec2_rule.delete(:user_id_group_pairs) if ec2_rule[:user_id_group_pairs].empty?
-
-              # if !ec2_rule[:user_id_group_pairs].nil? and
-                # ec2_rule[:user_id_group_pairs].size > 0 and
-                  # !ec2_rule[:ip_ranges].nil? and
-                  # ec2_rule[:ip_ranges].size > 0
-                # MU.log "Cannot specify ip_ranges and user_id_group_pairs", MU::ERR
-                # raise MuError, "Cannot specify ip_ranges and user_id_group_pairs"
-              # end
-
-              # if !ec2_rule[:user_id_group_pairs].nil? and
-                  # ec2_rule[:user_id_group_pairs].size > 0
-                # ec2_rule.delete(:ip_ranges)
-                # ec2_rule[:user_id_group_pairs].uniq!
-              # elsif !ec2_rule[:ip_ranges].nil? and
-                  # ec2_rule[:ip_ranges].size > 0
-                # ec2_rule.delete(:user_id_group_pairs)
-                # ec2_rule[:ip_ranges].uniq!
-              # end
-              ec2_rules << ec2_rule
+              }
+            end
+
+            ec2_rule[:user_id_group_pairs].uniq!
+            ec2_rule[:ip_ranges].uniq!
+            ec2_rule.delete(:ip_ranges) if ec2_rule[:ip_ranges].empty?
+            ec2_rule.delete(:user_id_group_pairs) if ec2_rule[:user_id_group_pairs].empty?
+
+            # if !ec2_rule[:user_id_group_pairs].nil? and
+              # ec2_rule[:user_id_group_pairs].size > 0 and
+                # !ec2_rule[:ip_ranges].nil? and
+                # ec2_rule[:ip_ranges].size > 0
+              # MU.log "Cannot specify ip_ranges and user_id_group_pairs", MU::ERR
+              # raise MuError, "Cannot specify ip_ranges and user_id_group_pairs"
+            # end
+
+            # if !ec2_rule[:user_id_group_pairs].nil? and
+                # ec2_rule[:user_id_group_pairs].size > 0
+              # ec2_rule.delete(:ip_ranges)
+              # ec2_rule[:user_id_group_pairs].uniq!
+            # elsif !ec2_rule[:ip_ranges].nil? and
+                # ec2_rule[:ip_ranges].size > 0
+              # ec2_rule.delete(:user_id_group_pairs)
+              # ec2_rule[:ip_ranges].uniq!
+            # end
+            ec2_rules << ec2_rule
+          }
+
+          # condense rules by protocol/port the way EC2 would read it back
+          grouped = {}
+          ec2_rules.each { |r|
+            key = r.dup
+            [:ip_ranges, :user_id_group_pairs].each { |t|
+              key.delete(t)
             }
-          end
+            grouped[key] ||= {}
+            [:ip_ranges, :user_id_group_pairs].each { |t|
+              next if !r[t]
+              grouped[key][t] ||= []
+              grouped[key][t].concat(r[t])
+              grouped[key][t].sort.uniq!
+            }
+          }
+          ec2_rules = grouped.keys.map { |k|
+            k.merge(grouped[k])
+          }
 
           ec2_rules.uniq
         end

data/modules/mu/providers/aws/function.rb

@@ -85,6 +85,7 @@ module MU
             changes[k] = v if v != old_props[k]
           }
           if !changes.empty?
+            wait_for_active
             MU.log "Updating Lambda #{@mu_name}", MU::NOTICE, details: changes
             MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).update_function_configuration(new_props)
           end
@@ -129,6 +130,25 @@ module MU
           
           end 
 
+          # If we have a loadbalancer configured, attach us to it
+          if !@config['loadbalancers'].nil?
+            if @loadbalancers.nil?
+              raise MuError, "#{@mu_name} is configured to use LoadBalancers, but none have been loaded by dependencies()"
+            end
+
+            @loadbalancers.each { |lb|
+              if !lb.targetgroups
+                MU.retrier([], max: 6, wait: 15, loop_if: Proc.new { !lb.targetgroups }) {
+                  lb.cloud_desc(use_cache: false)
+                }
+              end
+              lb.targetgroups.each_pair { |tg_name, tg|
+                addTrigger(tg.target_group_arn, "elasticloadbalancing", tg_name)
+              }
+              lb.registerTarget(arn)
+            }
+          end
+
           if @config['invoke_on_completion']
             invoke_params = {
               function_name: @cloud_id,
@@ -158,9 +178,34 @@ module MU
             statement_id: "#{calling_service}-#{calling_name.gsub(/[^a-z0-9\-_]/i, '_')}",
           }
 
+          # Just return if we already have this condition
+          begin
+            pol = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).get_policy(function_name: @cloud_id).policy
+            if pol
+              pol = JSON.parse(pol)
+              pol["Statement"].each { |s|
+                if !s["Condition"] or !s["Condition"]["ArnLike"] or
+                   !s["Condition"]["ArnLike"]["AWS:SourceArn"] or !s["Effect"] or
+                   !s["Principal"] or !s["Action"]
+                  next
+                end
+                if s["Effect"] == "Allow" and
+                   s["Sid"] == trigger[:statement_id] and
+                   s["Principal"] == { "Service" => trigger[:principal] } and
+                   s["Action"] == trigger[:action] and
+                   s["Condition"]["ArnLike"]["AWS:SourceArn"] == trigger[:source_arn]
+                  return
+
+                end
+              }
+            end
+          rescue Aws::Lambda::Errors::ResourceNotFoundException
+          end
+
           begin
             # XXX There doesn't seem to be an API call to list or view existing
             # permissions, wtaf. This means we can't intelligently guard this.
+            MU.log "Adding permission for Lambda function #{@cloud_id}", MU::NOTICE, details: trigger
             MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).add_permission(trigger)
           rescue Aws::Lambda::Errors::ValidationException => e
             MU.log e.message+" (calling_arn: #{calling_arn}, calling_service: #{calling_service}, calling_name: #{calling_name})", MU::ERR, details: trigger
@@ -416,7 +461,6 @@ module MU
 
           begin
             pol = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).get_policy(function_name: @cloud_id).policy
-MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV-2020080900-LN-ON-DEMAND-SCANNER"
             if pol
               bok['triggers'] ||= []
               JSON.parse(pol)["Statement"].each { |s|
@@ -619,7 +663,16 @@ MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV
           end
 
           if function['role']['name']
-            MU::Config.addDependency(function, function['role']['name'], "role")
+            MU::Config.addDependency(function, function['role']['name'], "role", my_phase: "groom", their_phase: "groom")
+          end
+
+          if !function["loadbalancers"].nil?
+            function["loadbalancers"].each { |lb|
+              lb["name"] ||= lb["concurrent_load_balancer"]
+              if lb["name"]
+                MU::Config.addDependency(function, lb["name"], "loadbalancer")
+              end
+            }
           end
 
           ok
@@ -627,6 +680,15 @@ MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV
 
         private
 
+        def wait_for_active
+          check = Proc.new {
+            resp = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).get_function(function_name: @cloud_id)
+            resp.configuration.state != "Active"
+          }
+          MU.retrier([], max: 40, wait: 15, loop_if: check) {
+          }
+        end
+
         def get_properties
           role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
           raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
@@ -724,7 +786,7 @@ MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV
               raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
             end
             lambda_properties[:vpc_config] = {
-              :subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
+              :subnet_ids => mySubnets.map { |s| s.cloud_id },
               :security_group_ids => sgs
             }
           end