cloud-mu 3.2.0 → 3.5.0

data/modules/mu/providers/aws/firewall_rule.rb

@@ -52,14 +52,14 @@ module MU
           begin
             MU.log "Creating EC2 Security Group #{groupname}", details: sg_struct
 
-            secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_security_group(sg_struct)
+            secgroup = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_security_group(sg_struct)
             @cloud_id = secgroup.group_id
           rescue Aws::EC2::Errors::InvalidGroupDuplicate
             MU.log "EC2 Security Group #{groupname} already exists, using it", MU::NOTICE
             filters = [{name: "group-name", values: [groupname]}]
             filters << {name: "vpc-id", values: [vpc_id]} if !vpc_id.nil?
 
-            secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(filters: filters).security_groups.first
+            secgroup = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_security_groups(filters: filters).security_groups.first
             if secgroup.nil?
               raise MuError, "Failed to locate security group named #{groupname}, even though EC2 says it already exists", caller
             end
@@ -67,25 +67,25 @@ module MU
           end
 
           begin
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(group_ids: [secgroup.group_id])
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_security_groups(group_ids: [secgroup.group_id])
           rescue Aws::EC2::Errors::InvalidGroupNotFound
             MU.log "#{secgroup.group_id} not yet ready, waiting...", MU::NOTICE
             sleep 10
             retry
           end
 
-          MU::Cloud::AWS.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
-          MU::Cloud::AWS.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
+          MU::Cloud::AWS.createStandardTags(secgroup.group_id, region: @region, credentials: @credentials)
+          MU::Cloud::AWS.createTag(secgroup.group_id, "Name", groupname, region: @region, credentials: @credentials)
 
           if @config['optional_tags']
             MU::MommaCat.listOptionalTags.each { |key, value|
-              MU::Cloud::AWS.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
+              MU::Cloud::AWS.createTag(secgroup.group_id, key, value, region: @region, credentials: @credentials)
             }
           end
 
           if @config['tags']
             @config['tags'].each { |tag|
-              MU::Cloud::AWS.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
+              MU::Cloud::AWS.createTag(secgroup.group_id, tag['key'], tag['value'], region: @region, credentials: @credentials)
             }
           end
 
@@ -123,7 +123,7 @@ module MU
         # Log metadata about this ruleset to the currently running deployment
         def notify
           sg_data = MU.structToHash(
-              MU::Cloud::FirewallRule.find(cloud_id: @cloud_id, region: @config['region'])
+              MU::Cloud::FirewallRule.find(cloud_id: @cloud_id, region: @region)
           )
           sg_data["group_id"] = @cloud_id
           sg_data["cloud_id"] = @cloud_id
@@ -151,8 +151,11 @@ module MU
             rule["firewall_rules"].concat(sgs.map { |s|
               MU::Config::Ref.get(
                 id: s,
+                region: @region,
+                credentials: @credentials,
                 cloud: "AWS",
-                type: "firewall_rule"
+                type: "firewall_rule",
+                dummy_ok: true
               )
             })
           end
@@ -169,12 +172,12 @@ module MU
 
           begin
             if egress
-              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
+              MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_egress(
                 group_id: @cloud_id,
                 ip_permissions: ec2_rule
               )
             else
-              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
+              MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_ingress(
                 group_id: @cloud_id,
                 ip_permissions: ec2_rule
               )
@@ -185,12 +188,12 @@ module MU
             # existing rules
             if comment
               if egress
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).update_security_group_rule_descriptions_egress(
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).update_security_group_rule_descriptions_egress(
                   group_id: @cloud_id,
                   ip_permissions: ec2_rule
                 )
               else
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).update_security_group_rule_descriptions_ingress(
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).update_security_group_rule_descriptions_ingress(
                   group_id: @cloud_id,
                   ip_permissions: ec2_rule
                 )
@@ -202,7 +205,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":security-group/"+@cloud_id
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":ec2:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":security-group/"+@cloud_id
         end
 
         # Locate an existing security group or groups and return an array containing matching AWS resource descriptors for those that match.
@@ -248,9 +251,9 @@ module MU
         def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
-            "credentials" => @config['credentials'],
+            "credentials" => @credentials,
             "cloud_id" => @cloud_id,
-            "region" => @config['region']
+            "region" => @region
           }
 
           if !cloud_desc
@@ -381,14 +384,14 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           filters = if flags and flags["vpc_id"]
             [
               {name: "vpc-id", values: [flags["vpc_id"]]}
             ]
           else
             filters = [
-              {name: "tag:MU-ID", values: [MU.deploy_id]}
+              {name: "tag:MU-ID", values: [deploy_id]}
             ]
             if !ignoremaster
               filters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
@@ -649,7 +652,7 @@ module MU
             if rule['firewall_rules']
               rule['firewall_rules'].each { |sg|
                 if sg['name'] and !sg['deploy_id']
-                  MU::Config.addDependency(acl, sg['name'], "firewall_rule", no_create_wait: true)
+                  MU::Config.addDependency(acl, sg['name'], "firewall_rule", my_phase: "groom")
                 end
               }
             end
@@ -657,7 +660,7 @@ module MU
             if rule['loadbalancers']
               rule['loadbalancers'].each { |lb|
                 if lb['name'] and !lb['deploy_id']
-                  MU::Config.addDependency(acl, lb['name'], "loadbalancer", phase: "groom")
+                  MU::Config.addDependency(acl, lb['name'], "loadbalancer", their_phase: "groom")
                 end
               }
             end
@@ -731,7 +734,7 @@ module MU
                 end
               }
               MU.log "Removing unconfigured rule in #{@mu_name}", MU::WARN, details: ext_rule
-              MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
+              MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_ingress(
                 group_id: @cloud_id,
                 ip_permissions: [ext_rule]
               )
@@ -797,7 +800,7 @@ module MU
               if ingress
                 if haverule
                   begin
-                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
+                    MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_ingress(
                       group_id: @cloud_id,
                       ip_permissions: [haverule]
                     )
@@ -805,7 +808,7 @@ module MU
                   end
                 end
                 begin
-                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
+                  MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_ingress(
                     group_id: @cloud_id,
                     ip_permissions: [rule]
                   )
@@ -818,14 +821,14 @@ module MU
               if egress
                 if haverule
                   begin
-                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
+                    MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).revoke_security_group_egress(
                       group_id: @cloud_id,
                       ip_permissions: [haverule]
                     )
                   rescue Aws::EC2::Errors::InvalidPermissionNotFound
                   end
                 end
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).authorize_security_group_egress(
                   group_id: @cloud_id,
                   ip_permissions: [rule]
                 )
@@ -860,8 +863,11 @@ module MU
                 p_start = rule['port'].to_i
                 p_end = rule['port'].to_i
               elsif rule['proto'] != "icmp"
-                raise MuError, "Can't create a TCP or UDP security group rule without specifying ports: #{rule}"
+                MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule
+                p_start = "0"
+                p_end = "65535"
               end
+
               if rule['proto'] != "icmp"
                 if p_start.nil? or p_end.nil?
                   raise MuError, "Got nil ports out of rule #{rule}"

data/modules/mu/providers/aws/folder.rb

@@ -59,7 +59,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
         end
 
         # Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.

data/modules/mu/providers/aws/function.rb

@@ -18,6 +18,18 @@ module MU
       # A function as configured in {MU::Config::BasketofKittens::functions}
       class Function < MU::Cloud::Function
 
+        # If we have sibling resources in our deployment, automatically inject
+        # interesting things about them into our function's environment
+        # variables.
+        SIBLING_VARS = {
+          "servers" => ["private_ip_address", "public_ip_address"],
+          "search_domains" => ["endpoint"],
+          "databases" => ["endpoint"],
+          "endpoints" => ["url"],
+          "notifiers" => ["TopicArn"],
+          "nosqldbs" => ["table_arn"]
+        }
+
         # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
         # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
         def initialize(**args)
@@ -26,10 +38,10 @@ module MU
         end
 
         # Tag this Lambda function
-        def assign_tag(resource_arn, tag_list, region=@config['region'])
+        def assign_tag(resource_arn, tag_list, region=@region)
           begin
             tag_list.each do |each_pair|
-              MU::Cloud::AWS.lambda(region: region, credentials: @config['credentials']).tag_resource({
+              MU::Cloud::AWS.lambda(region: region, credentials: @credentials).tag_resource({
                 resource: resource_arn,
                 tags: each_pair
               })
@@ -42,92 +54,47 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          role_arn = get_role_arn(@config['iam_role'])
-          
-          lambda_properties = {
-            code: {},
-            function_name: @mu_name,
-            handler: @config['handler'],
-            publish: true,
-            role: role_arn,
-            runtime: @config['runtime'],
-          }
-
-          if @config['code']['zip_file']
-            zip = File.read(@config['code']['zip_file'])
-            MU.log "Uploading deployment package from #{@config['code']['zip_file']}"
-            lambda_properties[:code][:zip_file] = zip
-          else
-            lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket']
-            lambda_properties[:code][:s3_key] = @config['code']['s3_key']
-            if @config['code']['s3_object_version']
-              lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version']
-            end
-          end
-           
-          if @config.has_key?('timeout')
-            lambda_properties[:timeout] = @config['timeout'].to_i ## secs
-          end           
-          
-          if @config.has_key?('memory')
-            lambda_properties[:memory_size] = @config['memory'].to_i
-          end
           
-          if @config.has_key?('environment_variables') 
-              lambda_properties[:environment] = { 
-                variables: {@config['environment_variables'][0]['key'] => @config['environment_variables'][0]['value']}
-              }
-          end
+          lambda_properties = get_properties
 
-          lambda_properties[:tags] = {}
-          MU::MommaCat.listStandardTags.each_pair { |k, v|
-            lambda_properties[:tags][k] = v
+          MU.retrier([Aws::Lambda::Errors::InvalidParameterValueException], max: 5, wait: 10) {
+            resp = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).create_function(lambda_properties)
+            @cloud_id = resp.function_name
           }
-          if @config['tags']
-            @config['tags'].each { |tag|
-              lambda_properties[:tags][tag.key.first] = tag.values.first
-            }
-          end
 
-          if @config.has_key?('vpc')
-            sgs = []
-            if @config['add_firewall_rules']
-              @config['add_firewall_rules'].each { |sg|
-                sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['name'])
-                sgs << sg.cloud_id if sg and sg.cloud_id
-              }
-            end
-            if !@vpc
-              raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
-            end
-            lambda_properties[:vpc_config] = {
-              :subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
-              :security_group_ids => sgs
-            }
-          end
-
-          retries = 0
-          resp = begin
-            MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
-          rescue Aws::Lambda::Errors::InvalidParameterValueException => e
-            # Freshly-made IAM roles sometimes aren't really ready
-            if retries < 5
-              sleep 10
-              retries += 1
-              retry
-            end
-            raise e
+          # the console does this and docs expect it to be there, so mimic the
+          # behavior
+          begin
+            MU::Cloud::AWS.cloudwatchlogs(region: @region, credentials: @credentials).create_log_group(
+              log_group_name: "/aws/lambda/#{@cloud_id}",
+              tags: @tags
+            )
+          rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
           end
-
-          @cloud_id = resp.function_name
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function(
-            function_name: @mu_name
-          )
-          func_arn = desc.configuration.function_arn if !desc.empty?
+          old_props = MU.structToHash(cloud_desc)
+
+          new_props = get_properties
+          code_block = new_props[:code]
+          new_props.reject! { |k, _v| [:code, :publish, :tags].include?(k) }
+          changes = {}
+          new_props.each_pair { |k, v|
+            changes[k] = v if v != old_props[k]
+          }
+          if !changes.empty?
+            MU.log "Updating Lambda #{@mu_name}", MU::NOTICE, details: changes
+            MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).update_function_configuration(new_props)
+          end
+
+          if @code_sha256 and @code_sha256 != cloud_desc.code_sha_256.chomp
+            MU.log "Updating code in Lambda #{@mu_name}", MU::NOTICE, details: { "old" => @code_sha256, "new" => cloud_desc.code_sha_256 }
+            code_block[:publish] = true
+            code_block[:function_name] = @cloud_id
+            MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).update_function_code(code_block)
+          end
 
 #          tag_function = assign_tag(lambda_func.function_arn, @config['tags']) 
           
@@ -141,7 +108,7 @@ module MU
           ### triggers must exist prior
           if @config['triggers']
             @config['triggers'].each { |tr|
-              trigger_arn = assume_trigger_arns(tr['service'], tr['name'])
+              trigger_arn = resolveARN(tr['service'], tr['name'])
 
               trigger_properties = {
                 action: "lambda:InvokeFunction", 
@@ -151,15 +118,33 @@ module MU
                 statement_id: "#{@mu_name}-ID-1",
               }
 
-              MU.log trigger_properties, MU::DEBUG
+              MU.log "Adding #{tr['service']} #{tr['name']} trigger to Lambda function #{@cloud_id}", details: trigger_properties
               begin
-                MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
+                MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).add_permission(trigger_properties)
               rescue Aws::Lambda::Errors::ResourceConflictException
+                # just means the permission is already there
               end
-              adjust_trigger(tr['service'], trigger_arn, func_arn, @mu_name) 
+              adjust_trigger(tr['service'], trigger_arn, arn, @mu_name) 
             }
           
           end 
+
+          if @config['invoke_on_completion']
+            invoke_params = {
+              function_name: @cloud_id,
+              invocation_type: @config['invoke_on_completion']['invocation_type'],
+              log_type: "Tail"
+            }
+            if @config['invoke_on_completion']['payload']
+              invoke_params[:payload] = JSON.generate(@config['invoke_on_completion']['payload'])
+            end
+            resp = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).invoke(invoke_params)
+            if resp.status_code == 200
+              MU.log "Invoked #{@cloud_id}", MU::NOTICE, details: Base64.decode64(resp.log_result)
+            else
+              MU.log "Invoked #{@cloud_id} and got #{resp.status_code} (#{resp.function_error})", MU::WARN, details: Base64.decode64(resp.log_result)
+            end
+          end
         end
 
         # Intended to be called by other Mu resources, such as Endpoints (API
@@ -170,16 +155,19 @@ module MU
             function_name: @mu_name, 
             principal: "#{calling_service}.amazonaws.com", 
             source_arn: calling_arn, 
-            statement_id: "#{calling_service}-#{calling_name}",
+            statement_id: "#{calling_service}-#{calling_name.gsub(/[^a-z0-9\-_]/i, '_')}",
           }
 
           begin
             # XXX There doesn't seem to be an API call to list or view existing
             # permissions, wtaf. This means we can't intelligently guard this.
-            MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger)
+            MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).add_permission(trigger)
+          rescue Aws::Lambda::Errors::ValidationException => e
+            MU.log e.message+" (calling_arn: #{calling_arn}, calling_service: #{calling_service}, calling_name: #{calling_name})", MU::ERR, details: trigger
+            raise e
           rescue Aws::Lambda::Errors::ResourceConflictException => e
             if e.message.match(/already exists/)
-              MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).remove_permission(
+              MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).remove_permission(
                 function_name: @mu_name,
                 statement_id: "#{calling_service}-#{calling_name}"
               )
@@ -192,17 +180,23 @@ module MU
         end
 
         # Look up an ARN for a given trigger type and resource name
-        def assume_trigger_arns(svc, name)
-          supported_triggers = %w(apigateway sns events event cloudwatch_event)
+        def resolveARN(svc, name)
+          supported_triggers = %w(apigateway sns events event cloudwatch_event dynamodb)
           if supported_triggers.include?(svc.downcase)
             arn = nil
             case svc.downcase
             when 'sns'
-              arn = "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
+              sib_sns = @deploy.findLitterMate(name: name, type: "notifiers")
+              arn = sib_sns ? sib_sns.arn : "arn:aws:sns:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:#{name}"
             when 'alarm','events', 'event', 'cloudwatch_event'
-              arn = "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}"
+              sib_event = @deploy.findLitterMate(name: name, type: "job")
+              arn = sib_event ? sib_event.arn : "arn:aws:events:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:rule/#{name}"
+            when 'dynamodb'
+              sib_dynamo = @deploy.findLitterMate(name: name, type: "nosqldb")
+              arn = sib_dynamo ? sib_dynamo.arn : "arn:aws:dynamodb:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:table/#{name}"
             when 'apigateway'
-              arn = "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
+              sib_apig = @deploy.findLitterMate(name: name, type: "endpoints")
+              arn = sib_apig ? sib_apig.arn : "arn:aws:apigateway:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:#{name}"
             when 's3'
               arn = ''
             end
@@ -214,21 +208,29 @@ module MU
         end
         
         # XXX placeholder, really; this is going end up being done from Endpoint, Log and Notification resources, I think
-        def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda',region=@config['region'])
+        def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda',region=@region)
           
           case trig_type
           
           when 'sns'
-           # XXX don't do this, use MU::Cloud::AWS::Notification 
-            sns_client = MU::Cloud::AWS.sns(region: region, credentials: @config['credentials'])
-            sns_client.subscribe({
-              topic_arn: trig_arn,
-              protocol: protocol,
-              endpoint: func_arn
-            })
+            MU::Cloud.resourceClass("AWS", "Notifier").subscribe(trig_arn, arn, "lambda", region: @region, credentials: @credentials)
+          when 'dynamodb'
+            stream = MU::Cloud::AWS.dynamostream(region: @region, credentials: @credentials).list_streams(table_name: trig_arn.sub(/.*?:table\//, '')).streams.first
+# XXX  guard this
+            MU.log "Adding DynamoDB Stream from #{stream.stream_arn} as trigger for #{@cloud_id}"
+            begin
+            MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).create_event_source_mapping(
+              event_source_arn: stream.stream_arn,
+              function_name: @cloud_id,
+              starting_position: "TRIM_HORIZON" # ...whatever that is
+            )
+            rescue ::Aws::Lambda::Errors::ResourceConflictException
+            end
+            
+#            MU::Cloud.resourceClass("AWS", "NoSQLDB").subscribe(trig_arn, arn, "lambda", region: @region, credentials: @credentials)
           when 'event','cloudwatch_event', 'events'
            # XXX don't do this, use MU::Cloud::AWS::Log
-            MU::Cloud::AWS.cloudwatch_events(region: region, credentials: @config['credentials']).put_targets({
+            MU::Cloud::AWS.cloudwatch_events(region: region, credentials: @credentials).put_targets({
               rule: @config['trigger']['name'],
               targets: [
                 {
@@ -237,9 +239,8 @@ module MU
                 }
               ]
             })
-#          when 'apigateway'
-# XXX this is actually happening in ::Endpoint... maybe...            
-#            MU.log "Creation of API Gateway integrations not yet implemented, you'll have to do this manually", MU::WARN, details: "(because we'll basically have to implement all of APIG for this)"
+          when 'apigateway'
+            addTrigger(trig_arn, "lambda", trig_arn.sub(/.*?([a-z0-9\-_]+)$/i, '\1'))
           end 
         end
 
@@ -247,9 +248,8 @@ module MU
         # Return the metadata for this Function rule
         # @return [Hash]
         def notify
-          deploy_struct = MU.structToHash(MU::Cloud::AWS::Function.find(cloud_id: @cloud_id, credentials: @config['credentials'], region: @config['region']).values.first)
-          deploy_struct['mu_name'] = @mu_name
-          return deploy_struct
+          return nil if !cloud_desc
+          MU.structToHash(cloud_desc, stringify_keys: true)
         end
 
         # Does this resource type exist as a global (cloud-wide) artifact, or
@@ -270,14 +270,14 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::Function.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
             desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
               function_name: f.function_name
             )
-            if desc.tags and desc.tags["MU-ID"] == MU.deploy_id and (desc.tags["MU-MASTER-IP"] == MU.mu_public_ip or ignoremaster)
+            if desc.tags and desc.tags["MU-ID"] == deploy_id and (desc.tags["MU-MASTER-IP"] == MU.mu_public_ip or ignoremaster)
               MU.log "Deleting Lambda function #{f.function_name}"
               if !noop
                 MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
@@ -292,7 +292,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          cloud_desc.function_arn
+          cloud_desc ? cloud_desc.function_arn : nil
         end
 
         # Locate an existing function.
@@ -317,9 +317,9 @@ module MU
         def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
-            "credentials" => @config['credentials'],
+            "credentials" => @credentials,
             "cloud_id" => @cloud_id,
-            "region" => @config['region']
+            "region" => @region
           }
 
           if !cloud_desc
@@ -333,7 +333,21 @@ module MU
           bok['runtime'] = cloud_desc.runtime
           bok['timeout'] = cloud_desc.timeout
 
-          function = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function(function_name: bok['name'])
+          function = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).get_function(function_name: bok['name'])
+#          event_srcs = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).list_event_source_mappings(function_name: @cloud_id)
+#          if event_srcs and !event_srcs.event_source_mappings.empty?
+#            MU.log "dem mappings tho #{@cloud_id}", MU::WARN, details: event_srcs
+#          end
+
+#          begin
+#            invoke_cfg = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).get_function_event_invoke_config(function_name: @cloud_id)
+#            MU.log "invoke config #{@cloud_id}", MU::WARN, details: invoke_cfg
+#          rescue ::Aws::Lambda::Errors::ResourceNotFoundException
+#          end
+
+#          MU.log @cloud_id, MU::WARN, details: cloud_desc if @cloud_id == "Espier-Scheduled-Scanner"
+#          MU.log "configuration #{@cloud_id}", MU::WARN, details: MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).get_function_configuration(function_name: @cloud_id) if @cloud_id == "Espier-Scheduled-Scanner"
+
 
           if function.code.repository_type == "S3"
             bok['code'] = {}
@@ -393,16 +407,29 @@ module MU
 
           if function.configuration.role
             shortname = function.configuration.role.sub(/.*?role\/([^\/]+)$/, '\1')
-MU.log shortname, MU::NOTICE, details: function.configuration.role
             bok['role'] = MU::Config::Ref.get(
               id: shortname,
-              name: shortname,
               cloud: "AWS",
               type: "roles"
             )
           end
+
+          begin
+            pol = MU::Cloud::AWS.lambda(region: @region, credentials: @credentials).get_policy(function_name: @cloud_id).policy
+MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV-2020080900-LN-ON-DEMAND-SCANNER"
+            if pol
+              bok['triggers'] ||= []
+              JSON.parse(pol)["Statement"].each { |s|
+                bok['triggers'] << {
+                  "service" => s["Principal"]["Service"].sub(/\..*/, ''),
+                  "name" => s["Resource"].sub(/.*?[:\/]([^:\/]+)$/, '\1')
+                }
+              }
+            end
+          rescue ::Aws::Lambda::Errors::ResourceNotFoundException
+          end
 #MU.log @cloud_id, MU::NOTICE, details: function
-# XXX triggers, permissions
+# XXX permissions
 
           bok
         end
@@ -414,6 +441,22 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
         def self.schema(_config)
           toplevel_required = ["runtime"]
           schema = {
+            "invoke_on_completion" => {
+              "type" => "object",
+              "description" => "Setting this will cause this Lambda function to be invoked when its groom phase is complete.",
+              "required" => ["invocation_type"],
+              "properties" => {
+                "invocation_type" => {
+                  "type" => "string",
+                  "enum" => ["RequestResponse", "Event", "Dryrun"],
+                  "default" => "RequestReponse"
+                },
+                "payload" => {
+                  "type" => "object",
+                  "description" => "Optional input to the function, which will be formatted as JSON and sent for execution"
+                }
+              }
+            },
             "triggers" => {
               "type" => "array",
               "items" => {
@@ -423,7 +466,7 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
                 "properties" => {
                   "service" => {
                     "type" => "string",
-                    "enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot},
+                    "enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot lex},
                     "description" => "The name of the AWS service that will trigger this function"
                   },
                   "name" => {
@@ -482,6 +525,28 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
         def self.validateConfig(function, configurator)
           ok = true
 
+          if function['triggers']
+            function['triggers'].each { |t|
+              mu_type = if t["service"] == "sns"
+                "notifiers"
+              elsif t["service"] == "apigateway"
+                "endpoints"
+              elsif t["service"] == "s3"
+                "buckets"
+              elsif t["service"] == "dynamodb"
+                "nosqldbs"
+              elsif t["service"] == "events"
+                "jobs"
+              elsif t["service"] == "sqs"
+                "msg_queues"
+              end
+
+              if mu_type
+                MU::Config.addDependency(function, t['name'], mu_type, my_phase: "groom")
+              end
+            }
+          end
+
           if function['vpc']
             fwname = "lambda-#{function['name']}"
             # default to allowing pings, if no ingress_rules were specified
@@ -508,7 +573,10 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
             MU::Config.addDependency(function, fwname, "firewall_rule")
           end
 
-          if !function['iam_role']
+          function['role'] ||= function['iam_role']
+          function.delete("iam_role")
+
+          if !function['role']
             policy_map = {
               "basic" => "AWSLambdaBasicExecutionRole",
               "kinesis" => "AWSLambdaKinesisExecutionRole",
@@ -537,9 +605,21 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
             }
             configurator.insertKitten(roledesc, "roles")
 
-            function['iam_role'] = function['name']+"execrole"
+            function['role'] = function['name']+"execrole"
 
-            MU::Config.addDependency(function, function['name']+"execrole", "role")
+          end
+
+          if function['role'].is_a?(String)
+            function['role'] = MU::Config::Ref.get(
+              name: function['role'],
+              type: "roles",
+              cloud: "AWS",
+              credentials: function['credentials']
+            )
+          end
+
+          if function['role']['name']
+            MU::Config.addDependency(function, function['role']['name'], "role")
           end
 
           ok
@@ -547,23 +627,109 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
 
         private
 
-        # Given an IAM role name, resolve to ARN. Will attempt to identify any
-        # sibling Mu role resources by this name first, and failing that, will
-        # do a plain get_role() to the IAM API for the provided name.
-        # @param name [String]
-        def get_role_arn(name)
-          sib_role = @deploy.findLitterMate(name: name, type: "roles")
-          return sib_role.cloudobj.arn if sib_role
+        def get_properties
+          role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
+          raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
 
-          begin
-            role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
-              role_name: name.to_s
-            })
-            return role['role']['arn']
-          rescue StandardError => e
-            MU.log "#{e}", MU::ERR
+          lambda_properties = {
+            code: {},
+            function_name: @mu_name,
+            handler: @config['handler'],
+            publish: true,
+            role: role_obj.arn,
+            runtime: @config['runtime'],
+          }
+
+          if @config['code']['zip_file'] or @config['code']['path']
+            tempfile = nil
+            if @config['code']['path']
+              tempfile = Tempfile.new
+              MU.log "#{@mu_name} using code at #{@config['code']['path']}"
+              MU::Master.zipDir(@config['code']['path'], tempfile.path)
+              @config['code']['zip_file'] = tempfile.path
+            else
+              MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
+            end
+            zip = File.read(@config['code']['zip_file'])
+            @code_sha256 = Base64.encode64(Digest::SHA256.digest(zip)).chomp
+            lambda_properties[:code][:zip_file] = zip
+            if tempfile
+              tempfile.close
+              tempfile.unlink
+            end
+          else
+            lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket']
+            lambda_properties[:code][:s3_key] = @config['code']['s3_key']
+            if @config['code']['s3_object_version']
+              lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version']
+            end
+# XXX need to download to a temporarily file, read it in, and calculate the digest in order to trigger updates in groom
           end
-          nil
+           
+          if @config.has_key?('timeout')
+            lambda_properties[:timeout] = @config['timeout'].to_i ## secs
+          end           
+          
+          if @config.has_key?('memory')
+            lambda_properties[:memory_size] = @config['memory'].to_i
+          end
+
+          SIBLING_VARS.each_key { |sib_type|
+            siblings = @deploy.findLitterMate(return_all: true, type: sib_type, cloud: "AWS")
+            if siblings
+              siblings.each_value { |sibling|
+                metadata = sibling.notify
+                if !metadata
+                  MU.log "Failed to extract metadata from sibling #{sibling}", MU::WARN
+                  next
+                end
+                SIBLING_VARS[sib_type].each { |var|
+                  if metadata[var]
+                    @config['environment_variables'] ||= []
+                    @config['environment_variables'] << {
+                      "key" => (sibling.config['name']+"_"+var).gsub(/[^a-z0-9_]/i, '_'),
+                      "value" => metadata[var]
+                    }
+                  end
+                }
+              }
+            end
+          }
+
+          if @config.has_key?('environment_variables') 
+            lambda_properties[:environment] = { 
+              variables: Hash[@config['environment_variables'].map { |v| [v['key'], v['value']] }]
+            }
+          end
+
+          lambda_properties[:tags] = {}
+          MU::MommaCat.listStandardTags.each_pair { |k, v|
+            lambda_properties[:tags][k] = v
+          }
+          if @config['tags']
+            @config['tags'].each { |tag|
+              lambda_properties[:tags][tag['key']] = tag['value']
+            }
+          end
+
+          if @config.has_key?('vpc')
+            sgs = []
+            if @config['add_firewall_rules']
+              @config['add_firewall_rules'].each { |sg|
+                sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['name'])
+                sgs << sg.cloud_id if sg and sg.cloud_id
+              }
+            end
+            if !@vpc
+              raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
+            end
+            lambda_properties[:vpc_config] = {
+              :subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
+              :security_group_ids => sgs
+            }
+          end
+
+          lambda_properties
         end
 
       end