jets.benforeva 3.0.17.pre.mount.pre.fix

data/lib/jets/job/base.rb

@@ -0,0 +1,49 @@
+require 'json'
+
+# Job public methods get turned into Lambda functions.
+#
+# Jets::Job::Base < Jets::Lambda::Functions
+# Both Jets::Job::Base and Jets::Lambda::Functions have Dsl modules included.
+# So the Jets::Job::Dsl overrides some of the Jets::Lambda::Functions behavior.
+module Jets::Job
+  class Base < Jets::Lambda::Functions
+    include Dsl
+
+    # non-DSL methods
+    include Helpers::KinesisEventHelper
+    include Helpers::LogEventHelper
+    include Helpers::S3EventHelper
+
+    # Tracks bucket each time an s3_event is declared
+    # Map of bucket_name => stack_name (nested part)
+    cattr_accessor :s3_events # dont want this to be inheritable intentionally
+    self.s3_events = {}
+
+    class << self
+      def process(event, context, meth)
+        job = new(event, context, meth)
+        job.send(meth)
+      end
+
+      def perform_now(meth, event={}, context={})
+        new(event.try(:to_unsafe_h), context, meth).send(meth)
+      end
+
+      def perform_later(meth, event={}, context={})
+        if on_lambda?
+          function_name = "#{self.to_s.underscore}-#{meth}"
+          call = Jets::Commands::Call.new(function_name, JSON.dump(event.try(:to_unsafe_h)), invocation_type: "Event")
+          call.run
+        else
+          puts "INFO: Not on AWS Lambda. In local mode perform_later executes the job with perform_now instead."
+          perform_now(meth, event, context)
+        end
+      end
+
+    private
+      def on_lambda?
+        !!ENV['AWS_LAMBDA_FUNCTION_NAME']
+      end
+    end
+  end
+end

data/lib/jets/job/dsl/dynamodb_event.rb

@@ -0,0 +1,67 @@
+module Jets::Job::Dsl
+  module DynamodbEvent
+    def dynamodb_event(table_name, options={})
+      return if ENV['JETS_BUILD_NO_INTERNET'] # Disable during build since jets build tries to init this
+
+      stream_arn = full_dynamodb_stream_arn(table_name)
+      default_iam_policy = default_dynamodb_stream_policy(stream_arn)
+
+      # Create iam policy allows access to queue
+      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
+      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
+      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+
+      props = options # by this time options only has EventSourceMapping properties
+      default = {
+        event_source_arn: stream_arn,
+        starting_position: "TRIM_HORIZON",
+      }
+      props = default.merge(props)
+
+      event_source_mapping(props)
+    end
+
+
+    # Expands table name to the full stream arn. Example:
+    #
+    #   test-table
+    # To:
+    #   arn:aws:dynamodb:us-west-2:112233445566:table/test-table/stream/2019-02-15T21:41:15.217
+    #
+    # Note, this does not check if the stream has been disabled.
+    def full_dynamodb_stream_arn(table_name)
+      return table_name if table_name.include?("arn:aws:dynamodb") # assume full stream arn
+
+      begin
+        resp = dynamodb.describe_table(table_name: table_name)
+      rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
+        puts e.message
+        puts "ERROR: Was not able to find the DynamoDB table: #{table_name}.".color(:red)
+        code_line = caller.grep(%r{/app/jobs}).first
+        puts "Please check: #{code_line}"
+        puts "Exiting"
+        exit 1
+      end
+      stream_arn = resp.table.latest_stream_arn
+      return stream_arn if stream_arn
+    end
+
+    def default_dynamodb_stream_policy(stream_name_arn='*')
+      stream = {
+        action: ["dynamodb:GetRecords",
+                 "dynamodb:GetShardIterator",
+                 "dynamodb:DescribeStream",
+                 "dynamodb:ListStreams"],
+        effect: "Allow",
+        resource: stream_name_arn,
+      }
+      table_name_arn = stream_name_arn.gsub(%r{/stream/20.*},'')
+      table = {
+        action: ["dynamodb:DescribeTable"],
+        effect: "Allow",
+        resource: table_name_arn,
+      }
+      [stream, table]
+    end
+  end
+end

data/lib/jets/job/dsl/event_source_mapping.rb

@@ -0,0 +1,11 @@
+# SqsEvent uses this module
+module Jets::Job::Dsl
+  module EventSourceMapping
+    def event_source_mapping(props={})
+      r = Jets::Resource::Lambda::EventSourceMapping.new(props)
+      with_fresh_properties do
+        resource(r.definition) # add associated resource immediately
+      end
+    end
+  end
+end

data/lib/jets/job/dsl/iot_event.rb

@@ -0,0 +1,24 @@
+module Jets::Job::Dsl
+  module IotEvent
+    # The user must at least pass in an SQL statement
+    def iot_event(props={})
+      if props.is_a?(String) # SQL Statement
+        props = {sql: props}
+        topic_props = {topic_rule_payload: props}
+      elsif props.key?(:topic_rule_payload) # full properties structure
+        topic_props = props
+      else # just the topic_rule_payload
+        topic_props = {topic_rule_payload: props}
+      end
+
+      declare_iot_topic(topic_props)
+    end
+
+    def declare_iot_topic(props={})
+      r = Jets::Resource::Iot::TopicRule.new(props)
+      with_fresh_properties do
+        resource(r.definition) # add associated resource immediately
+      end
+    end
+  end
+end

data/lib/jets/job/dsl/kinesis_event.rb

@@ -0,0 +1,46 @@
+module Jets::Job::Dsl
+  module KinesisEvent
+    def kinesis_event(stream_name, options={})
+      stream_arn = full_kinesis_stream_arn(stream_name)
+      default_iam_policy = default_kinesis_stream_policy(stream_arn)
+
+      # Create iam policy allows access to queue
+      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
+      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
+      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+
+      props = options # by this time options only has EventSourceMapping properties
+      default = {
+        event_source_arn: stream_arn,
+        starting_position: "LATEST",
+      }
+      props = default.merge(props)
+
+      event_source_mapping(props)
+    end
+
+    # Expands table name to the full stream arn. Example:
+    #
+    #   test-table
+    # To:
+    #   arn:aws:kinesis:us-west-2:112233445566:table/test-table/stream/2019-02-15T21:41:15.217
+    #
+    # Note, this does not check if the stream has been disabled.
+    def full_kinesis_stream_arn(stream_name)
+      return stream_name if stream_name.include?("arn:aws:kinesis") # assume full stream arn
+
+      "arn:aws:kinesis:#{Jets.aws.region}:#{Jets.aws.account}:stream/#{stream_name}"
+    end
+
+    def default_kinesis_stream_policy(stream_name_arn='*')
+      {
+        action: ["kinesis:GetRecords",
+                 "kinesis:GetShardIterator",
+                 "kinesis:DescribeStream",
+                 "kinesis:ListStreams"],
+        effect: "Allow",
+        resource: stream_name_arn,
+      }
+    end
+  end
+end

data/lib/jets/job/dsl/log_event.rb

@@ -0,0 +1,15 @@
+module Jets::Job::Dsl
+  module LogEvent
+    def log_event(log_group_name, props={})
+      props.merge!(log_group_name: log_group_name)
+      declare_log_subscription_filter(props)
+    end
+
+    def declare_log_subscription_filter(props={})
+      r = Jets::Resource::Logs::SubscriptionFilter.new(props)
+      with_fresh_properties do
+        resource(r.definition) # add associated resource immediately
+      end
+    end
+  end
+end

data/lib/jets/job/dsl/rule_event.rb

@@ -0,0 +1,75 @@
+module Jets::Job::Dsl
+  module RuleEvent
+    # Public: Creates CloudWatch Event Rule
+    #
+    # expression - The rate expression.
+    #
+    # Examples
+    #
+    #   rate("10 minutes")
+    #   rate("10 minutes", description: "Hard job")
+    #
+    def rate(expression, props={})
+      schedule_job("rate(#{expression})", props)
+    end
+
+    # Public: Creates CloudWatch Event Rule
+    #
+    # expression - The cron expression.
+    #
+    # Examples
+    #
+    #   cron("0 */12 * * ? *")
+    #   cron("0 */12 * * ? *", description: "Hard job")
+    #
+    def cron(expression, props={})
+      schedule_job("cron(#{expression})", props)
+    end
+
+    def schedule_job(expression, props={})
+      props = props.merge(schedule_expression: expression)
+      rule_event(props)
+    end
+
+    def rule_event(props={})
+      if props.key?(:detail)
+        description = props.key?(:description) ? props.delete(:description) : rule_description
+        rule_props = { event_pattern: props, description: description }
+      else # if props.key?(:event_pattern)
+        props[:description] ||= rule_description
+        rule_props = props
+      end
+
+      with_fresh_properties(multiple_resources: false) do
+        associated_properties(rule_props) # TODO: consider getting rid of @associated_properties concept
+        resource(events_rule_definition) # add associated resource immediately
+      end
+    end
+
+    def rule_description
+      self.rule_counter += 1
+      "#{self.name} event rule #{rule_counter}"
+    end
+
+    def events_rule_definition
+      resource = Jets::Resource::Events::Rule.new(associated_properties)
+      resource.definition # returns a definition to be added by associated_resources
+    end
+
+    # Deprecated methods, will be removed in the future
+    def events_rule(props)
+      puts "DEPRECATED: events_rule. Instead use rule_event. The events_rule will be removed in the future.  Pausing for 5 seconds".color(:yellow)
+      puts caller[0]
+      sleep 5
+      rule_event(props)
+    end
+    # Deprecated methods, will be removed in the future
+
+    def event_pattern(props)
+      puts "DEPRECATED: events_rule. Instead use rule_event. The events_rule will be removed in the future.  Pausing for 5 seconds".color(:yellow)
+      puts caller[0]
+      sleep 5
+      rule_event(props)
+    end
+  end
+end

data/lib/jets/job/dsl/s3_event.rb

@@ -0,0 +1,36 @@
+module Jets::Job::Dsl
+  module S3Event
+    def s3_event(bucket_name, props={})
+      stack_name = declare_s3_bucket_resources(bucket_name) # only set up once per bucket
+      declare_sns_subscription(topic_arn: "!Ref #{stack_name}SnsTopic") # set up subscription every time
+    end
+
+    # Returns stack_name
+    def declare_s3_bucket_resources(bucket_name)
+      # If shared s3 bucket resources have already been declared.
+      # We will not generate them again. However, we still need to always
+      # add the depends_on declaration to ensure that the shared stack parameters
+      # are properly passed to the nested child stack.
+      stack_name = s3_events[bucket_name] # already registered
+      if stack_name
+        depends_on stack_name.underscore.to_sym, class_prefix: true # always add this
+        return stack_name
+      end
+
+      # Create shared resources - one time
+      stack_name = declare_shared_s3_event_resources(bucket_name)
+      depends_on stack_name.underscore.to_sym, class_prefix: true # always add this
+      self.s3_events[bucket_name] = stack_name # tracks buckets already set up
+    end
+
+    def declare_shared_s3_event_resources(bucket_name)
+      s3_stack = Jets::Stack::S3Event.new(bucket_name)
+      s3_stack.build_stack
+      s3_stack.stack_name
+    end
+
+    def s3_events
+      Jets::Job::Base.s3_events
+    end
+  end
+end

data/lib/jets/job/dsl/sns_event.rb

@@ -0,0 +1,54 @@
+module Jets::Job::Dsl
+  module SnsEvent
+    def sns_event(topic_name, props={})
+      if topic_name.to_s =~ /generate/
+        declare_sns_topic(props.delete(:topic_properties))
+        topic_arn = "!Ref {namespace}SnsTopic"
+        props.merge!(topic_arn: topic_arn)
+        declare_sns_subscription(props)
+      elsif topic_name.include?('!Ref') # reference shared resource
+        topic_arn = topic_name # contains !Ref
+        props.merge!(topic_arn: topic_arn)
+        declare_sns_subscription(props)
+      else # existing topic: short name or full arn
+        topic_arn = full_sns_topic_arn(topic_name)
+        props.merge!(topic_arn: topic_arn)
+        declare_sns_subscription(props)
+      end
+    end
+
+    def declare_sns_topic(props={})
+      props ||= {} # props.delete(:topic_properties) can be nil
+      r = Jets::Resource::Sns::Topic.new(props)
+      with_fresh_properties do
+        resource(r.definition) # add associated resource immediately
+      end
+    end
+
+    def declare_sns_topic_policy(props={})
+      props ||= {} # options.delete(:topic_policy_properties) can be nil
+      r = Jets::Resource::Sns::TopicPolicy.new(props)
+      with_fresh_properties do
+        resource(r.definition) # add associated resource immediately
+      end
+    end
+
+    def declare_sns_subscription(props={})
+      r = Jets::Resource::Sns::Subscription.new(props)
+      with_fresh_properties do
+        resource(r.definition) # add associated resource immediately
+      end
+    end
+
+    # Expands simple topic name to full arn. Example:
+    #
+    #   hello-topic
+    # To:
+    #   arn:aws:sns:us-west-2:112233445566:hello-topic
+    def full_sns_topic_arn(topic_name)
+      return topic_name if topic_name.include?("arn:aws:sns")
+
+      "arn:aws:sns:#{Jets.aws.region}:#{Jets.aws.account}:#{topic_name}"
+    end
+  end
+end

data/lib/jets/job/dsl/sqs_event.rb

@@ -0,0 +1,96 @@
+# ## Event Source Mapping
+#
+# Underneath the hood, the `sqs_event` method sets up a  [Lambda::EventSourceMapping](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html).  So:
+#
+# ```ruby
+# class HardJob
+#   class_timeout 30 # must be less than or equal to the SQS queue default timeout
+#
+#   sqs_event "hello-queue"
+#   def dig
+#     puts "dig event #{JSON.dump(event)}"
+#   end
+# end
+# ```
+#
+# Cloud also be written with something like this:
+#
+# ```ruby
+# class HardJob
+#   class_timeout 30 # must be less than or equal to the SQS queue default timeout
+#
+#   event_source_mapping(
+#     event_source_arn: "arn:aws:sqs:us-west-2:112233445566:hello-queue",
+#   )
+#   iam_policy(
+#     action: ["sqs:ReceiveMessage",
+#             "sqs:DeleteMessage",
+#             "sqs:GetQueueAttributes"],
+#     effect: "Allow",
+#     resource: "arn:aws:sqs:us-west-2:112233445566:hello-queue",
+#   )
+#   def dig
+#     puts "dig event #{JSON.dump(event)}"
+#   end
+# end
+# ```
+#
+module Jets::Job::Dsl
+  module SqsEvent
+    def sqs_event(queue_name, options={})
+      if queue_name.to_s =~ /generate/
+        queue_arn = "!GetAtt {namespace}SqsQueue.Arn"
+        default_iam_policy = default_sqs_iam_policy('*') # Dont have access to full ARN on initial creation
+        declare_sqs_queue(options.delete(:queue_properties)) # delete to avoid using them for event_source_mapping
+      elsif queue_name.include?('!Ref') # reference shared resource
+        queue_arn = queue_name
+        default_iam_policy = default_sqs_iam_policy('*') # Dont have access to full ARN on initial creation
+      else # existing queue: short name or full arn
+        queue_arn = full_sqs_queue_arn(queue_name)
+        default_iam_policy = default_sqs_iam_policy(queue_arn)
+      end
+
+      # Create iam policy allows access to queue
+      # Allow disabling in case use wants to add permission application-wide and not have extra IAM policy
+      iam_policy_props = options.delete(:iam_policy) || @iam_policy || default_iam_policy
+      iam_policy(iam_policy_props) unless iam_policy_props == :disable
+
+      props = options # by this time options only has EventSourceMapping properties
+      default = {
+        event_source_arn: queue_arn
+      }
+      props = default.merge(props)
+
+      event_source_mapping(props)
+    end
+
+    def declare_sqs_queue(props)
+      props ||= {} # since options.delete(:queue_properties) can be nil
+      r = Jets::Resource::Sqs::Queue.new(props)
+      with_fresh_properties do
+        resource(r.definition) # add associated resource immediately
+      end
+    end
+
+    # Expands simple queue name to full arn. Example:
+    #
+    #   hello-queue
+    # To:
+    #   arn:aws:sqs:us-west-2:112233445566:hello-queue
+    def full_sqs_queue_arn(queue_name)
+      return queue_name if queue_name.include?("arn:aws:sqs")
+
+      "arn:aws:sqs:#{Jets.aws.region}:#{Jets.aws.account}:#{queue_name}"
+    end
+
+    def default_sqs_iam_policy(queue_name_arn='*')
+      {
+        action: ["sqs:ReceiveMessage",
+                 "sqs:DeleteMessage",
+                 "sqs:GetQueueAttributes"],
+        effect: "Allow",
+        resource: queue_name_arn,
+      }
+    end
+  end
+end

data/lib/jets/job/dsl.rb

@@ -0,0 +1,40 @@
+require 'active_support'
+require 'active_support/core_ext/class'
+
+# Jets::Job::Base < Jets::Lambda::Functions
+# Both Jets::Job::Base and Jets::Lambda::Functions have Dsl modules included.
+# So the Jets::Job::Dsl overrides some of the Jets::Lambda::Functions behavior.
+#
+# Implements:
+#
+#   default_associated_resource_definition
+#
+module Jets::Job::Dsl
+  extend ActiveSupport::Concern
+
+  included do
+    class << self
+      include Jets::AwsServices
+
+      include DynamodbEvent
+      include EventSourceMapping
+      include IotEvent
+      include KinesisEvent
+      include LogEvent
+      include RuleEvent
+      include S3Event
+      include SnsEvent
+      include SqsEvent
+
+      # Used to provide a little more identifiable event rule auto-descriptions
+      class_attribute :rule_counter
+      self.rule_counter = 0
+
+      # TODO: Get rid of default_associated_resource_definition concept.
+      # Also gets rid of the need to keep track of running @associated_properties too.
+      def default_associated_resource_definition(meth)
+        events_rule_definition
+      end
+    end
+  end
+end

data/lib/jets/job/helpers/kinesis_event_helper.rb

@@ -0,0 +1,13 @@
+require 'base64'
+
+module Jets::Job::Helpers
+  module KinesisEventHelper
+    def kinesis_data
+      records = event["Records"]
+      records.map do |record|
+        encoded = record["kinesis"]["data"]
+        Base64.decode64(encoded) # data
+      end
+    end
+  end
+end

data/lib/jets/job/helpers/log_event_helper.rb

@@ -0,0 +1,17 @@
+require 'base64'
+require 'json'
+require 'stringio'
+require 'zlib'
+
+module Jets::Job::Helpers
+  module LogEventHelper
+    def log_event
+      encoded = event["awslogs"]["data"]
+      compressed_string = Base64.decode64(encoded)
+      gz = Zlib::GzipReader.new(StringIO.new(compressed_string))
+      uncompressed_string = gz.read
+      data = JSON.load(uncompressed_string)
+      ActiveSupport::HashWithIndifferentAccess.new(data)
+    end
+  end
+end

data/lib/jets/job/helpers/s3_event_helper.rb

@@ -0,0 +1,13 @@
+module Jets::Job::Helpers
+  module S3EventHelper
+    def s3_event
+      message = event["Records"][0]["Sns"]["Message"]
+      h = JSON.load(message)
+      ActiveSupport::HashWithIndifferentAccess.new(h)
+    end
+
+    def s3_object
+      s3_event["Records"][0]["s3"]["object"]
+    end
+  end
+end

data/lib/jets/klass.rb

@@ -0,0 +1,109 @@
+# Loading a class can usually be loaded via .constantize.
+# But app/functions files are anonymous ruby classes created with
+# Class.new.  Anonymous classes cannot be loaded via .constantize and
+# go through standard autoloading.
+#
+# Jets::Klass provides a way to load app classes in app/controllers, app/jobs,
+# app/functions in a consistent way without having to worry about the anonymous
+# class loading quirk. Classes that are not anonymously defined like controllers
+# and jobs are loaded via autoloading with .constantize. Anonymously defined
+# classes like functions are loaded via Object.const_set.
+#
+# Examples:
+#
+#   Jets::Klass.from_path("app/controllers/posts_controller.rb")
+#   Jets::Klass.from_path("app/jobs/hard_job.rb")
+#   Jets::Klass.from_path("app/functions/hello.rb")
+#   Jets::Klass.from_path("app/functions/hello_function.rb")
+#   Jets::Klass.from_path("app/shared/functions/whatever.rb")
+#
+#   Jets::Klass.from_task(task)
+#
+# The from_task method takes a Jets::Lambda::Task as an argument and is useful
+# for the CloudFormation child stack generation there the registered task info
+# is available but the path info is now.
+class Jets::Klass
+  class << self
+    # from_path allows us to load any app classes in consistent way for
+    # app/controllers, app/jobs, and app/functions.
+    def from_path(path)
+      class_name = class_name(path)
+      if path.include?("/functions/") # simple function
+        class_name = load_anonymous_class(class_name, path)
+        class_name.constantize # removed :: for anonymous classes
+      else
+        class_name.constantize # autoload
+      end
+    end
+
+    # app/controllers/posts_controller.rb => PostsController
+    def class_name(path)
+      if path.include?("/shared/")
+        path.sub(%r{.*app/shared/(.*?)/},'').sub(/\.rb$/,'').camelize
+      else
+        path.sub(%r{.*app/(.*?)/},'').sub(/\.rb$/,'').camelize
+      end
+    end
+
+    APP_TYPES = %w[controller job rule authorizer]
+    def from_task(task)
+      class_name = task.class_name
+      filename = class_name.underscore
+
+      # Examples of filename: posts_controller, hard_job, security_rule, main_authorizer
+      #   hello_function, hello
+      type = filename.split('_').last
+      type = "function" unless APP_TYPES.include?(type)
+
+      path = "app/#{type.pluralize}/#{filename}.rb"
+      from_path(path)
+    end
+
+    @@loaded_anonymous_classes = []
+    def load_anonymous_class(class_name, path)
+      parent_mod = modularize(class_name)
+
+      constructor = Jets::Lambda::FunctionConstructor.new(path)
+      # Dont load anonyomous class more than once to avoid these warnings:
+      #   warning: already initialized constant Hello
+      #   warning: previous definition of Hello was here
+      unless @@loaded_anonymous_classes.include?(class_name)
+        # use class_name as the variable name for prettier class name.
+        leaf_class_name = class_name.split('::').last
+        parent_mod.const_set(leaf_class_name, constructor.build)
+        @@loaded_anonymous_classes << class_name
+      end
+
+      class_name
+    end
+
+    # Ensures the parent namespace modules are defined. Example:
+    #
+    #   modularize("Foo::Bar::Test")
+    #   => Foo::Bar # is a now defined as a module if it wasnt before
+    #
+    # Also returns the parent module, so we can use it to do a const_set if needed. IE:
+    #
+    #   parent_mod = modularize("Foo::Bar::Test")
+    #   parent_mod.const_set("Test")
+    def modularize(class_name)
+      leaves = []
+      mods = class_name.split('::')[0..-2] # drop the last word
+      # puts "mods: #{mods}"
+      return Object if mods.empty?
+
+      leaves = []
+      mods.each do |leaf_mod|
+        leaves += [leaf_mod]
+        namespace = leaves.join('::')
+        previous_namespace = leaves[0..-2].join('::')
+        previous_namespace = "Object" if previous_namespace.empty?
+        previous_namespace = previous_namespace.constantize
+        previous_namespace.const_set(leaf_mod, Module.new) unless Object.const_defined?(namespace)
+      end
+
+      mods.join('::').constantize
+    end
+
+  end
+end