jets 3.1.0 → 5.0.10

data/lib/jets/job/dsl/dynamodb_event.rb

@@ -1,26 +1,37 @@
 module Jets::Job::Dsl
   module DynamodbEvent
-    def dynamodb_event(table_name, options={})
-      return if ENV['JETS_BUILD_NO_INTERNET'] # Disable during build since jets build tries to init this
+    def dynamodb_event(table_name_without_namespace, options={})
+      return if ENV['JETS_NO_INTERNET'] # Disable during build since jets build tries to init this
 
+      table_name = add_dynamodb_table_namespace(table_name_without_namespace)
       stream_arn = full_dynamodb_stream_arn(table_name)
       default_iam_policy = default_dynamodb_stream_policy(stream_arn)
 
-      # Create iam policy allows access to queue
-      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
-      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
-      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+      # Create iam policy allows access to dynamodb
+      iam_policy_option = [options.delete(:iam_policy)].compact.flatten
+      user_iam_policy = [@iam_policy].compact.flatten
+      iam_policy_props = iam_policy_option + user_iam_policy + [default_iam_policy]
+      iam_policy(iam_policy_props)
 
       props = options # by this time options only has EventSourceMapping properties
       default = {
-        event_source_arn: stream_arn,
-        starting_position: "TRIM_HORIZON",
+        EventSourceArn: stream_arn,
+        StartingPosition: "TRIM_HORIZON",
       }
       props = default.merge(props)
 
       event_source_mapping(props)
     end
 
+    def add_dynamodb_table_namespace(table_name_without_namespace)
+      ns = if Jets.config.events.dynamodb.table_namespace == true
+             Jets.table_namespace # does not include extra
+           elsif Jets.config.events.dynamodb.table_namespace
+             Jets.config.events.dynamodb.table_namespace # allow user to fully control namespace
+           end
+      ns_separator = Jets.config.events.dynamodb.table_namespace_separator
+      [ns, table_name_without_namespace].compact.join(ns_separator)
+    end
 
     # Expands table name to the full stream arn. Example:
     #
@@ -48,18 +59,18 @@ module Jets::Job::Dsl
 
     def default_dynamodb_stream_policy(stream_name_arn='*')
       stream = {
-        action: ["dynamodb:GetRecords",
+        Action: ["dynamodb:GetRecords",
                  "dynamodb:GetShardIterator",
                  "dynamodb:DescribeStream",
                  "dynamodb:ListStreams"],
-        effect: "Allow",
-        resource: stream_name_arn,
+        Effect: "Allow",
+        Resource: stream_name_arn,
       }
       table_name_arn = stream_name_arn.gsub(%r{/stream/20.*},'')
       table = {
-        action: ["dynamodb:DescribeTable"],
-        effect: "Allow",
-        resource: table_name_arn,
+        Action: ["dynamodb:DescribeTable"],
+        Effect: "Allow",
+        Resource: table_name_arn,
       }
       [stream, table]
     end

data/lib/jets/job/dsl/event_source_mapping.rb

@@ -2,7 +2,7 @@
 module Jets::Job::Dsl
   module EventSourceMapping
     def event_source_mapping(props={})
-      r = Jets::Resource::Lambda::EventSourceMapping.new(props)
+      r = Jets::Cfn::Resource::Lambda::EventSourceMapping.new(props)
       with_fresh_properties do
         resource(r.definition) # add associated resource immediately
       end

data/lib/jets/job/dsl/iot_event.rb

@@ -3,19 +3,19 @@ module Jets::Job::Dsl
     # The user must at least pass in an SQL statement
     def iot_event(props={})
       if props.is_a?(String) # SQL Statement
-        props = {sql: props}
-        topic_props = {topic_rule_payload: props}
-      elsif props.key?(:topic_rule_payload) # full properties structure
+        props = {Sql: props}
+        topic_props = {TopicRulePayload: props}
+      elsif props.key?(:TopicRulePayload) # full properties structure
         topic_props = props
-      else # just the topic_rule_payload
-        topic_props = {topic_rule_payload: props}
+      else # just the TopicRulePayload
+        topic_props = {TopicRulePayload: props}
       end
 
       declare_iot_topic(topic_props)
     end
 
     def declare_iot_topic(props={})
-      r = Jets::Resource::Iot::TopicRule.new(props)
+      r = Jets::Cfn::Resource::Iot::TopicRule.new(props)
       with_fresh_properties do
         resource(r.definition) # add associated resource immediately
       end

data/lib/jets/job/dsl/kinesis_event.rb

@@ -4,15 +4,16 @@ module Jets::Job::Dsl
       stream_arn = full_kinesis_stream_arn(stream_name)
       default_iam_policy = default_kinesis_stream_policy(stream_arn)
 
-      # Create iam policy allows access to queue
-      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
-      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
-      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+      # Create iam policy allows access to kinesis
+      iam_policy_option = [options.delete(:iam_policy)].compact.flatten
+      user_iam_policy = [@iam_policy].compact.flatten
+      iam_policy_props = iam_policy_option + user_iam_policy + [default_iam_policy]
+      iam_policy(iam_policy_props)
 
       props = options # by this time options only has EventSourceMapping properties
       default = {
-        event_source_arn: stream_arn,
-        starting_position: "LATEST",
+        EventSourceArn: stream_arn,
+        StartingPosition: "LATEST",
       }
       props = default.merge(props)
 
@@ -34,12 +35,12 @@ module Jets::Job::Dsl
 
     def default_kinesis_stream_policy(stream_name_arn='*')
       {
-        action: ["kinesis:GetRecords",
+        Action: ["kinesis:GetRecords",
                  "kinesis:GetShardIterator",
                  "kinesis:DescribeStream",
                  "kinesis:ListStreams"],
-        effect: "Allow",
-        resource: stream_name_arn,
+        Effect: "Allow",
+        Resource: stream_name_arn,
       }
     end
   end

data/lib/jets/job/dsl/log_event.rb

@@ -1,12 +1,12 @@
 module Jets::Job::Dsl
   module LogEvent
     def log_event(log_group_name, props={})
-      props.merge!(log_group_name: log_group_name)
+      props.merge!(LogGroupName: log_group_name)
       declare_log_subscription_filter(props)
     end
 
     def declare_log_subscription_filter(props={})
-      r = Jets::Resource::Logs::SubscriptionFilter.new(props)
+      r = Jets::Cfn::Resource::Logs::SubscriptionFilter.new(props)
       with_fresh_properties do
         resource(r.definition) # add associated resource immediately
       end

data/lib/jets/job/dsl/rule_event.rb

@@ -27,16 +27,19 @@ module Jets::Job::Dsl
     end
 
     def schedule_job(expression, props={})
-      props = props.merge(schedule_expression: expression)
+      props = props.merge(ScheduleExpression: expression)
       rule_event(props)
     end
 
     def rule_event(props={})
+      # detail should all be lowercase: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html
+      # source should also all be lowercase
       if props.key?(:detail)
+        # :description should be lowercased. It's a convenience property, and belongs at a higher level
         description = props.key?(:description) ? props.delete(:description) : rule_description
-        rule_props = { event_pattern: props, description: description }
+        rule_props = { EventPattern: props, Description: description }
       else # if props.key?(:event_pattern)
-        props[:description] ||= rule_description
+        props[:Description] ||= rule_description
         rule_props = props
       end
 
@@ -52,7 +55,7 @@ module Jets::Job::Dsl
     end
 
     def events_rule_definition
-      resource = Jets::Resource::Events::Rule.new(associated_properties)
+      resource = Jets::Cfn::Resource::Events::Rule.new(associated_properties)
       resource.definition # returns a definition to be added by associated_resources
     end
 

data/lib/jets/job/dsl/s3_event.rb

@@ -2,7 +2,7 @@ module Jets::Job::Dsl
   module S3Event
     def s3_event(bucket_name, props={})
       stack_name = declare_s3_bucket_resources(bucket_name) # only set up once per bucket
-      declare_sns_subscription(topic_arn: "!Ref #{stack_name}SnsTopic") # set up subscription every time
+      declare_sns_subscription(TopicArn: "!Ref #{stack_name}SnsTopic") # set up subscription every time
     end
 
     # Returns stack_name
@@ -11,7 +11,7 @@ module Jets::Job::Dsl
       # We will not generate them again. However, we still need to always
       # add the depends_on declaration to ensure that the shared stack parameters
       # are properly passed to the nested child stack.
-      stack_name = s3_events[bucket_name] # already registered
+      stack_name = _s3_events[bucket_name] # already registered
       if stack_name
         depends_on stack_name.underscore.to_sym, class_prefix: true # always add this
         return stack_name
@@ -20,7 +20,7 @@ module Jets::Job::Dsl
       # Create shared resources - one time
       stack_name = declare_shared_s3_event_resources(bucket_name)
       depends_on stack_name.underscore.to_sym, class_prefix: true # always add this
-      self.s3_events[bucket_name] = stack_name # tracks buckets already set up
+      self._s3_events[bucket_name] = stack_name # tracks buckets already set up
     end
 
     def declare_shared_s3_event_resources(bucket_name)
@@ -29,8 +29,8 @@ module Jets::Job::Dsl
       s3_stack.stack_name
     end
 
-    def s3_events
-      Jets::Job::Base.s3_events
+    def _s3_events
+      Jets::Job::Base._s3_events
     end
   end
 end

data/lib/jets/job/dsl/sns_event.rb

@@ -4,22 +4,22 @@ module Jets::Job::Dsl
       if topic_name.to_s =~ /generate/
         declare_sns_topic(props.delete(:topic_properties))
         topic_arn = "!Ref {namespace}SnsTopic"
-        props.merge!(topic_arn: topic_arn)
+        props.merge!(TopicArn: topic_arn)
         declare_sns_subscription(props)
       elsif topic_name.include?('!Ref') # reference shared resource
         topic_arn = topic_name # contains !Ref
-        props.merge!(topic_arn: topic_arn)
+        props.merge!(TopicArn: topic_arn)
         declare_sns_subscription(props)
       else # existing topic: short name or full arn
         topic_arn = full_sns_topic_arn(topic_name)
-        props.merge!(topic_arn: topic_arn)
+        props.merge!(TopicArn: topic_arn)
         declare_sns_subscription(props)
       end
     end
 
     def declare_sns_topic(props={})
       props ||= {} # props.delete(:topic_properties) can be nil
-      r = Jets::Resource::Sns::Topic.new(props)
+      r = Jets::Cfn::Resource::Sns::Topic.new(props)
       with_fresh_properties do
         resource(r.definition) # add associated resource immediately
       end
@@ -27,14 +27,14 @@ module Jets::Job::Dsl
 
     def declare_sns_topic_policy(props={})
       props ||= {} # options.delete(:topic_policy_properties) can be nil
-      r = Jets::Resource::Sns::TopicPolicy.new(props)
+      r = Jets::Cfn::Resource::Sns::TopicPolicy.new(props)
       with_fresh_properties do
         resource(r.definition) # add associated resource immediately
       end
     end
 
     def declare_sns_subscription(props={})
-      r = Jets::Resource::Sns::Subscription.new(props)
+      r = Jets::Cfn::Resource::Sns::Subscription.new(props)
       with_fresh_properties do
         resource(r.definition) # add associated resource immediately
       end

data/lib/jets/job/dsl/sqs_event.rb

@@ -20,14 +20,14 @@
 #   class_timeout 30 # must be less than or equal to the SQS queue default timeout
 #
 #   event_source_mapping(
-#     event_source_arn: "arn:aws:sqs:us-west-2:112233445566:hello-queue",
+#     EventSourceArn: "arn:aws:sqs:us-west-2:112233445566:hello-queue",
 #   )
 #   iam_policy(
-#     action: ["sqs:ReceiveMessage",
-#             "sqs:DeleteMessage",
-#             "sqs:GetQueueAttributes"],
-#     effect: "Allow",
-#     resource: "arn:aws:sqs:us-west-2:112233445566:hello-queue",
+#     Action: ["sqs:ReceiveMessage",
+#              "sqs:DeleteMessage",
+#              "sqs:GetQueueAttributes"],
+#     Effect: "Allow",
+#     Resource: "arn:aws:sqs:us-west-2:112233445566:hello-queue",
 #   )
 #   def dig
 #     puts "dig event #{JSON.dump(event)}"
@@ -51,13 +51,14 @@ module Jets::Job::Dsl
       end
 
       # Create iam policy allows access to queue
-      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
-      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
-      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+      iam_policy_option = [options.delete(:iam_policy)].compact.flatten
+      user_iam_policy = [@iam_policy].compact.flatten
+      iam_policy_props = iam_policy_option + user_iam_policy + [default_iam_policy]
+      iam_policy(iam_policy_props)
 
       props = options # by this time options only has EventSourceMapping properties
       default = {
-        event_source_arn: queue_arn
+        EventSourceArn: queue_arn
       }
       props = default.merge(props)
 
@@ -66,7 +67,7 @@ module Jets::Job::Dsl
 
     def declare_sqs_queue(props)
       props ||= {} # since options.delete(:queue_properties) can be nil
-      r = Jets::Resource::Sqs::Queue.new(props)
+      r = Jets::Cfn::Resource::Sqs::Queue.new(props)
       with_fresh_properties do
         resource(r.definition) # add associated resource immediately
       end
@@ -85,11 +86,11 @@ module Jets::Job::Dsl
 
     def default_sqs_iam_policy(queue_name_arn='*')
       {
-        action: ["sqs:ReceiveMessage",
+        Action: ["sqs:ReceiveMessage",
                  "sqs:DeleteMessage",
                  "sqs:GetQueueAttributes"],
-        effect: "Allow",
-        resource: queue_name_arn,
+        Effect: "Allow",
+        Resource: queue_name_arn,
       }
     end
   end

data/lib/jets/job/helpers/{kinesis_event_helper.rb → kinesis_event.rb}

@@ -1,7 +1,7 @@
 require 'base64'
 
 module Jets::Job::Helpers
-  module KinesisEventHelper
+  module KinesisEvent
     def kinesis_data
       records = event["Records"]
       records.map do |record|
@@ -9,5 +9,9 @@ module Jets::Job::Helpers
         Base64.decode64(encoded) # data
       end
     end
+
+    def kinesis_data?
+      event["Records"]&.any? { |r| r.dig("kinesis", "data") }
+    end
   end
 end

data/lib/jets/job/helpers/{log_event_helper.rb → log_event.rb}

@@ -4,7 +4,7 @@ require 'stringio'
 require 'zlib'
 
 module Jets::Job::Helpers
-  module LogEventHelper
+  module LogEvent
     def log_event
       encoded = event["awslogs"]["data"]
       compressed_string = Base64.decode64(encoded)
@@ -13,5 +13,9 @@ module Jets::Job::Helpers
       data = JSON.load(uncompressed_string)
       ActiveSupport::HashWithIndifferentAccess.new(data)
     end
+
+    def log_event?
+      !!event.dig("awslogs", "data")
+    end
   end
 end

data/lib/jets/job/helpers/s3_event.rb

@@ -0,0 +1,50 @@
+module Jets::Job::Helpers
+  module S3Event
+    def s3_events
+      encoded_messages = event["Records"].map do |record|
+        record["Sns"]["Message"] # SNS message is JSON
+      end
+      # Decode the JSON messages
+      messages = encoded_messages.map do |message|
+        JSON.load(message)
+      end
+      # Extract the S3 event records
+      messages.map do |message|
+        message["Records"].map do |record|
+          ActiveSupport::HashWithIndifferentAccess.new(record)
+        end
+      end.flatten
+    end
+    alias s3_event_payloads s3_events
+
+    def s3_events?
+      event["Records"]&.any? { |r| r.dig("Sns", "Message") }
+    end
+    alias s3_event_payloads? s3_events?
+
+    def s3_objects
+      s3_events.map do |record|
+        record["s3"]["object"]
+      end
+    end
+
+    def s3_objects?
+      s3_events.any? { |r| r.dig("s3", "object") }
+    end
+
+    # Deprecated methods below
+    def s3_event
+      puts "WARN: s3_event is deprecated".color(:yellow)
+      puts "It can possibly drop events when they come in extremely fast."
+      puts "Use s3_events instead"
+      s3_events.first
+    end
+
+    def s3_object
+      puts "WARN: s3_object is deprecated".color(:yellow)
+      puts "It can possibly drop events when they come in extremely fast."
+      puts "Use s3_objects instead"
+      s3_objects.first
+    end
+  end
+end

data/lib/jets/job/helpers/sns_event.rb

@@ -0,0 +1,26 @@
+module Jets::Job::Helpers
+  module SnsEvent
+    def sns_events
+      records = event["Records"]
+      return [] unless records
+      records.map do |record|
+        message = record["Sns"]["Message"]
+        ActiveSupport::HashWithIndifferentAccess.new(JSON.load(message))
+      end
+    end
+    alias sns_event_payloads sns_events
+
+    def sns_events?
+      event["Records"]&.any? { |r| r.dig("Sns", "Message") }
+    end
+    alias sns_event_payloads? sns_events?
+
+    # Deprecated methods below
+    def sns_event_payload
+      puts "WARN: sns_event_payload is deprecated".color(:yellow)
+      puts "It can possibly drop events when they come in extremely fast."
+      puts "Use sns_events instead"
+      sns_events.first
+    end
+  end
+end

data/lib/jets/job/helpers/sqs_event.rb

@@ -0,0 +1,26 @@
+module Jets::Job::Helpers
+  module SqsEvent
+    def sqs_events
+      records = event["Records"]
+      return [] unless records
+      records.map do |record|
+        message = record["body"]
+        ActiveSupport::HashWithIndifferentAccess.new(JSON.load(message))
+      end
+    end
+    alias sqs_event_payloads sqs_events
+
+    def sqs_events?
+      event["Records"]&.any? { |r| r.dig("body") }
+    end
+    alias sqs_event_payloads? sqs_events?
+
+    # Deprecated methods below
+    def sqs_event_payload
+      puts "WARN: sqs_event_payload is deprecated".color(:yellow)
+      puts "It can possibly drop events when they come in extremely fast."
+      puts "Use sqs_events instead"
+      sqs_events.first
+    end
+  end
+end

data/lib/jets/klass.rb

@@ -17,10 +17,10 @@
 #   Jets::Klass.from_path("app/functions/hello_function.rb")
 #   Jets::Klass.from_path("app/shared/functions/whatever.rb")
 #
-#   Jets::Klass.from_task(task)
+#   Jets::Klass.from_definition(definition)
 #
-# The from_task method takes a Jets::Lambda::Task as an argument and is useful
-# for the CloudFormation child stack generation there the registered task info
+# The from_definition method takes a Jets::Lambda::Definition as an argument and is useful
+# for the CloudFormation child stack generation there the registered definition info
 # is available but the path info is now.
 class Jets::Klass
   class << self
@@ -46,8 +46,8 @@ class Jets::Klass
     end
 
     APP_TYPES = %w[controller job rule authorizer]
-    def from_task(task)
-      class_name = task.class_name
+    def from_definition(definition)
+      class_name = definition.class_name
       filename = class_name.underscore
 
       # Examples of filename: posts_controller, hard_job, security_rule, main_authorizer
@@ -89,7 +89,6 @@ class Jets::Klass
     def modularize(class_name)
       leaves = []
       mods = class_name.split('::')[0..-2] # drop the last word
-      # puts "mods: #{mods}"
       return Object if mods.empty?
 
       leaves = []

data/lib/jets/lambda/definition.rb

@@ -0,0 +1,118 @@
+module Jets::Lambda
+  class Definition
+    include Jets::Util::Camelize
+
+    attr_accessor :class_name, :type
+    attr_reader :meth, :properties, :iam_policy, :managed_iam_policy, :lang, :associated_resources
+    def initialize(class_name, meth, options={})
+      @class_name = class_name.to_s
+      @meth = meth
+      @options = options
+      @type = options[:type] || get_type  # controller, job, or function
+      @properties = camelize(options[:properties]) || {}
+      @iam_policy = options[:iam_policy]
+      @managed_iam_policy = options[:managed_iam_policy]
+      @lang = options[:lang] || :ruby
+      @associated_resources = options[:associated_resources] || {}
+      @replacements = options[:replacements] || {} # added to baseline replacements
+    end
+
+    def name
+      @meth
+    end
+
+    def public_meth?
+      # For anonymous classes (app/functions/hello.rb) the class name will be blank.
+      # These types of classes are treated specially and has only one handler method
+      # that is registered. So we know it is public.
+      return true if @class_name.nil? || @class_name == ''
+
+      # Consider all polymorphic methods public for now
+      return true if @lang != :ruby
+
+      klass = @class_name.constantize
+      public_methods = klass.public_instance_methods
+      public_methods.include?(meth.to_sym)
+    end
+
+    def build_function_iam?
+      !!(@iam_policy || @managed_iam_policy)
+    end
+
+    @@lang_exts = {
+      ruby: '.rb',
+      python: '.py',
+      node: '.js',
+    }
+    def lang_ext
+      @@lang_exts[@lang]
+    end
+
+    # The get_type method works for controller and job classes.
+    #
+    # Usually able to get the type from the class name. Examples:
+    #
+    #   PostsController => controller
+    #   HardJob => job
+    #
+    # However, for function types, we are not able to get the type for multiple of
+    # reasons.  First, function types are allowed to be named with or without
+    # _function.  Examples:
+    #
+    #   path => class => type
+    #   app/functions/hello.rb => Hello => function
+    #   app/functions/hello_function.rb => HelloFunction => function
+    #
+    # The second reason is that functions are not regular ruby classes. Instead they
+    # are anonymous classes created with Class.new.  When classes are created with
+    # Class.new the method_added hook has "" (blank string) as the self class name.
+    # We add the class_type to the task later on as we are constructing the class
+    # as part of the Class.new logic.
+    #
+    # For controller and job standard ruby classes though it can easily be
+    # determinated as part of initialization. So we get the type for convenience then.
+    #
+    # For anonymous function classes, we just set to nil and will later fix in
+    # FunctionConstructor.
+    #
+    # Returns: "controller", "job" or nil
+    def get_type
+      unless @class_name.empty? # when anonymous class is created with Class.new
+        @class_name.underscore.split('_').last # controller, job or rule
+      end
+    end
+
+    def full_handler(handler_function)
+      "#{handler_base}.#{handler_function}"
+    end
+
+    def handler_path
+      "#{handler_base}#{lang_ext}"
+    end
+
+    def handler_base
+      base = "handlers/#{@type.pluralize}/#{@class_name.underscore}"
+      base += "/#{@lang}" if @lang != :ruby
+      base += "/#{@meth}"
+    end
+
+    def poly_src_path
+      handler_path.sub("handlers/", "app/")
+    end
+
+    def replacements
+      # Merge in the custom replacements specific to each app class: ConfigRule, Job, etc.
+      base_replacements.merge(@replacements)
+    end
+
+    def base_replacements
+      { namespace: namespace }
+    end
+
+    def namespace
+      ns = @class_name.gsub('::','').camelize
+      ns += @meth.to_s.camelize if @meth
+      ns
+    end
+  end
+end