aws-sdk-rails 3.1.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c6f3ca6246615704132c2a690056d8c8636fd40947f756e79318affd6148720
-  data.tar.gz: e17f171dba5c857c58358a6e35d962b398ada43439c78c8e0df496ed342dd1ef
+  metadata.gz: 49d723b9b6fd27217d47fec5d67a8e215eea7249b190dcd724d949539c4fccdc
+  data.tar.gz: 0d99a46f3109fa5ea2950045a4e24ea85f630c9458d510bffe208dca65fcce22
 SHA512:
-  metadata.gz: 61effb18895b3c4de7aa1cb8e44de99f5c497ac0ff996e5d74ae99af1b363dc634bb753e2c2160d437ede22988cdcb145b6683989864ee42b4cc029487649a0c
-  data.tar.gz: 3b7ebe7488bd432776e1c594dc0325b7d75a964276d7e5af1ab05bbb66f3139de581ccc8c77730d97d69b8b39d5ab0312cb9e5f1fb719b9c5da5f0ce9dde1c5b
+  metadata.gz: 3411b20bf529a39dc0c3f45259f5b470cf74885a8c87937c5f874714ba0f840c33e6609f9904c673d0879bacddb30e1f8d9d2f8173ca229a7d4d264be6880e05
+  data.tar.gz: e2e4626edffb9ee1be443330ec975875ac97569034d985a551f24557d33c3c09cc6120f4f47a48bad15673f3481d52d5544349f8e2f109127a0714f0354d1b41

data/VERSION

@@ -0,0 +1 @@
+3.6.0

data/bin/aws_sqs_active_job

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/aws/rails/sqs_active_job/poller'
+
+Aws::Rails::SqsActiveJob::Poller.new.run

data/lib/action_dispatch/session/dynamodb_store.rb

@@ -0,0 +1,32 @@
+require 'aws-sessionstore-dynamodb'
+
+module ActionDispatch
+  module Session
+    # Uses the Dynamo DB Session Store implementation to create a class that
+    # extends ActionDispatch::Session. Rails will create a :dynamodb_store
+    # configuration for session_store from this class name.
+    #
+    # This class will use the Rails secret_key_base unless otherwise provided.
+    #
+    # Configuration can also be provided in YAML files from Rails config, either
+    # in "config/session_store.yml" or "config/session_store/#{Rails.env}.yml".
+    # Configuration files that are environment-specific will take precedence.
+    #
+    # @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
+    class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
+      def initialize(app, options = {})
+        options[:config_file] ||= config_file if config_file.exist?
+        options[:secret_key] ||= Rails.application.secret_key_base
+        super
+      end
+
+      private
+
+      def config_file
+        file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
+        file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
+        file
+      end
+    end
+  end
+end

data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+module ActiveJob
+  module QueueAdapters
+
+    class AmazonSqsAdapter
+
+      def enqueue(job)
+        _enqueue(job)
+      end
+
+      def enqueue_at(job, timestamp, opts={})
+        delay = (timestamp - Time.now.to_f).floor
+        raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
+        _enqueue(job, delay_seconds: delay)
+      end
+
+      private
+
+      def _enqueue(job, send_message_opts = {})
+        body = job.serialize
+        queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
+        send_message_opts[:queue_url] = queue_url
+        send_message_opts[:message_body] = Aws::Json.dump(body)
+        send_message_opts[:message_attributes] = message_attributes(job)
+
+        if Aws::Rails::SqsActiveJob.fifo?(queue_url)
+          # job_id is unique per initialization of job
+          # Remove it from message dup id to ensure run-once behavior
+          # with ActiveJob retries
+           send_message_opts[:message_deduplication_id] =
+             Digest::SHA256.hexdigest(
+               Aws::Json.dump(body.except('job_id'))
+             )
+
+           send_message_opts[:message_group_id] = Aws::Rails::SqsActiveJob.config.message_group_id
+        end
+        Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
+      end
+
+      def message_attributes(job)
+        {
+          'aws_sqs_active_job_class' => {
+            string_value: job.class.to_s,
+            data_type: 'String'
+          },
+          'aws_sqs_active_job_version' => {
+            string_value: Aws::Rails::VERSION,
+            data_type: 'String'
+          }
+        }
+      end
+    end
+
+    # create an alias to allow `:amazon` to be used as the adapter name
+    # `:amazon` is the convention used for ActionMailer and ActiveStorage
+    AmazonAdapter = AmazonSqsAdapter
+  end
+end

data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'concurrent'
+
+module ActiveJob
+  module QueueAdapters
+
+    # == Async adapter for Amazon SQS ActiveJob
+    #
+    # This adapter queues jobs asynchronously (ie non-blocking).  Error handler can be configured
+    # with +Aws::Rails::SqsActiveJob.config.async_queue_error_handler+.
+    #
+    # To use this adapter, set up as:
+    #
+    # config.active_job.queue_adapter = :amazon_sqs_async
+    class AmazonSqsAsyncAdapter < AmazonSqsAdapter
+
+      private
+
+      def _enqueue(job, send_message_opts = {})
+        # FIFO jobs must be queued in order, so do not queue async
+        queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
+        if Aws::Rails::SqsActiveJob.fifo?(queue_url)
+          super(job, send_message_opts)
+        else
+          Concurrent::Promise
+          .execute { super(job, send_message_opts) }
+          .on_error do |e|
+            Rails.logger.error "Failed to queue job #{job}.  Reason: #{e}"
+            error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
+            error_handler.call(e, job, send_message_opts) if error_handler
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-rails.rb

@@ -1,51 +1,22 @@
 # frozen_string_literal: true
 
 require_relative 'aws/rails/mailer'
+require_relative 'aws/rails/railtie'
+require_relative 'aws/rails/notifications'
+require_relative 'aws/rails/sqs_active_job/configuration'
+require_relative 'aws/rails/sqs_active_job/executor'
+require_relative 'aws/rails/sqs_active_job/job_runner'
+require_relative 'aws/rails/sqs_active_job/lambda_handler'
+require_relative 'aws/rails/middleware/ebs_sqs_active_job_middleware'
 
-module Aws
-  # Use the Rails namespace.
-  module Rails
-    # @api private
-    class Railtie < ::Rails::Railtie
-      initializer 'aws-sdk-rails.initialize',
-                  before: :load_config_initializers do
-        # Initialization Actions
-        Aws::Rails.use_rails_encrypted_credentials
-        Aws::Rails.add_action_mailer_delivery_method
-        Aws::Rails.log_to_rails_logger
-      end
-    end
-
-    # This is called automatically from the SDK's Railtie, but can be manually
-    # called if you want to specify options for building the Aws::SES::Client.
-    #
-    # @param [Symbol] name The name of the ActionMailer delivery method to
-    #   register.
-    # @param [Hash] options The options you wish to pass on to the
-    #   Aws::SES::Client initialization method.
-    def self.add_action_mailer_delivery_method(name = :ses, options = {})
-      ActiveSupport.on_load(:action_mailer) do
-        add_delivery_method(name, Aws::Rails::Mailer, options)
-      end
-    end
+require_relative 'action_dispatch/session/dynamodb_store'
+require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
+require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
 
-    # Configures the AWS SDK for Ruby's logger to use the Rails logger.
-    def self.log_to_rails_logger
-      Aws.config[:logger] = ::Rails.logger
-      nil
-    end
+require_relative 'generators/aws_record/base'
 
-    # Configures the AWS SDK with credentials from Rails encrypted credentials.
-    def self.use_rails_encrypted_credentials
-      # limit the config keys we merge to credentials only
-      aws_credential_keys = %i[access_key_id secret_access_key session_token]
-
-      Aws.config.merge!(
-        ::Rails.application
-          .try(:credentials)
-          .try(:aws)
-          .to_h.slice(*aws_credential_keys)
-      )
-    end
+module Aws
+  module Rails
+    VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
   end
 end

data/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    # Middleware to handle requests from the SQS Daemon present on Elastic Beanstalk worker environments.
+    class EbsSqsActiveJobMiddleware
+      INTERNAL_ERROR_MESSAGE = 'Failed to execute job - see Rails log for more details.'
+      INTERNAL_ERROR_RESPONSE = [500, { 'Content-Type' => 'text/plain' }, [INTERNAL_ERROR_MESSAGE]].freeze
+      FORBIDDEN_MESSAGE = 'Request with aws-sqsd user agent was made from untrusted address.'
+      FORBIDDEN_RESPONSE = [403, { 'Content-Type' => 'text/plain' }, [FORBIDDEN_MESSAGE]].freeze
+
+      def initialize(app)
+        @app = app
+        @logger = ActiveSupport::Logger.new(STDOUT)
+      end
+
+      def call(env)
+        request = ActionDispatch::Request.new(env)
+
+        # Pass through unless user agent is the SQS Daemon
+        return @app.call(env) unless from_sqs_daemon?(request)
+
+        @logger.debug('aws-rails-sdk middleware detected call from Elastic Beanstalk SQS Daemon.')
+
+        # Only accept requests from this user agent if it is from localhost or a docker host in case of forgery.
+        unless request.local? || sent_from_docker_host?(request)
+          @logger.warn("SQSD request detected from untrusted address #{request.remote_ip}; returning 403 forbidden.")
+          return FORBIDDEN_RESPONSE
+        end
+
+        # Execute job or periodic task based on HTTP request context
+        periodic_task?(request) ? execute_periodic_task(request) : execute_job(request)
+      end
+
+      private
+
+      def execute_job(request)
+        # Jobs queued from the Active Job SQS adapter contain the JSON message in the request body.
+        job = Aws::Json.load(request.body.string)
+        job_name = job['job_class']
+        @logger.debug("Executing job: #{job_name}")
+
+        begin
+          ActiveJob::Base.execute(job)
+        rescue NoMethodError, NameError => e
+          @logger.error("Job #{job_name} could not resolve to a class that inherits from Active Job.")
+          @logger.error("Error: #{e}")
+          return INTERNAL_ERROR_RESPONSE
+        end
+
+        [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran job #{job_name}."]]
+      end
+
+      def execute_periodic_task(request)
+        # The beanstalk worker SQS Daemon will add the 'X-Aws-Sqsd-Taskname' for periodic tasks set in cron.yaml.
+        job_name = request.headers['X-Aws-Sqsd-Taskname']
+        @logger.debug("Creating and executing periodic task: #{job_name}")
+
+        begin
+          job = job_name.constantize.new
+          job.perform_now
+        rescue NoMethodError, NameError => e
+          @logger.error("Periodic task #{job_name} could not resolve to an Active Job class - check the spelling in cron.yaml.")
+          @logger.error("Error: #{e}.")
+          return INTERNAL_ERROR_RESPONSE
+        end
+
+        [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran periodic task #{job_name}."]]
+      end
+
+      # The beanstalk worker SQS Daemon sets a specific User-Agent headers that begins with 'aws-sqsd'.
+      def from_sqs_daemon?(request)
+        current_user_agent = request.headers['User-Agent']
+
+        !current_user_agent.nil? && current_user_agent.start_with?('aws-sqsd')
+      end
+
+      # The beanstalk worker SQS Daemon will add the custom 'X-Aws-Sqsd-Taskname' header for periodic tasks set in cron.yaml.
+      def periodic_task?(request)
+        !request.headers['X-Aws-Sqsd-Taskname'].nil? && request.headers['X-Aws-Sqsd-Taskname'].present?
+      end
+
+      def sent_from_docker_host?(request)
+        app_runs_in_docker_container? && request.remote_ip == '172.17.0.1'
+      end
+
+      def app_runs_in_docker_container?
+        @app_runs_in_docker_container ||= `[ -f /proc/1/cgroup ] && cat /proc/1/cgroup` =~ /docker/
+      end
+    end
+  end
+end

data/lib/aws/rails/notifications.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-core'
+require 'active_support/notifications'
+
+module Aws
+  module Rails
+
+    # Instruments client operation calls for ActiveSupport::Notifications
+    # Each client operation will produce an event with name:
+    # <operation>.<service>.aws
+    # @api private
+    class Notifications < Seahorse::Client::Plugin
+
+      def add_handlers(handlers, config)
+        # This plugin needs to be first
+        # which means it is called first in the stack, to start recording time,
+        # and returns last
+        handlers.add(Handler, step: :initialize, priority: 99)
+      end
+
+      class Handler < Seahorse::Client::Handler
+
+        def call(context)
+          event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
+          ActiveSupport::Notifications.instrument(event_name, context: context) do
+            @handler.call(context)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/railtie.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Aws
+  # Use the Rails namespace.
+  module Rails
+    # @api private
+    class Railtie < ::Rails::Railtie
+      initializer 'aws-sdk-rails.initialize',
+                  before: :load_config_initializers do
+        # Initialization Actions
+        Aws::Rails.use_rails_encrypted_credentials
+        Aws::Rails.add_action_mailer_delivery_method
+        Aws::Rails.log_to_rails_logger
+      end
+
+      initializer 'aws-sdk-rails.insert_middleware' do |app|
+        Aws::Rails.add_sqsd_middleware(app)
+      end
+
+      rake_tasks do
+        load 'tasks/dynamo_db/session_store.rake'
+        load 'tasks/aws_record/migrate.rake'
+      end
+    end
+
+    # This is called automatically from the SDK's Railtie, but can be manually
+    # called if you want to specify options for building the Aws::SES::Client.
+    #
+    # @param [Symbol] name The name of the ActionMailer delivery method to
+    #   register.
+    # @param [Hash] options The options you wish to pass on to the
+    #   Aws::SES::Client initialization method.
+    def self.add_action_mailer_delivery_method(name = :ses, options = {})
+      ActiveSupport.on_load(:action_mailer) do
+        add_delivery_method(name, Aws::Rails::Mailer, options)
+      end
+    end
+
+    # Configures the AWS SDK for Ruby's logger to use the Rails logger.
+    def self.log_to_rails_logger
+      Aws.config[:logger] = ::Rails.logger
+      nil
+    end
+
+    # Configures the AWS SDK with credentials from Rails encrypted credentials.
+    def self.use_rails_encrypted_credentials
+      # limit the config keys we merge to credentials only
+      aws_credential_keys = %i[access_key_id secret_access_key session_token]
+
+      Aws.config.merge!(
+        ::Rails.application
+          .try(:credentials)
+          .try(:aws)
+          .to_h.slice(*aws_credential_keys)
+      )
+    end
+
+    # Adds ActiveSupport Notifications instrumentation to AWS SDK
+    # client operations.  Each operation will produce an event with a name:
+    # <operation>.<service>.aws.  For example, S3's put_object has an event
+    # name of: put_object.S3.aws
+    def self.instrument_sdk_operations
+      Aws.constants.each do |c|
+        m = Aws.const_get(c)
+        if m.is_a?(Module) && m.const_defined?(:Client) &&
+           m.const_get(:Client).superclass == Seahorse::Client::Base
+          m.const_get(:Client).add_plugin(Aws::Rails::Notifications)
+        end
+      end
+    end
+
+    # Register a middleware that will handle requests from the Elastic Beanstalk worker SQS Daemon.
+    # This will only be added in the presence of the AWS_PROCESS_BEANSTALK_WORKER_REQUESTS environment variable.
+    # The expectation is this variable should only be set on EB worker environments.
+    def self.add_sqsd_middleware(app)
+      is_eb_worker_hosted = Aws::Util.str_2_bool(ENV['AWS_PROCESS_BEANSTALK_WORKER_REQUESTS'].to_s.downcase)
+
+      return unless is_eb_worker_hosted
+
+      if app.config.force_ssl
+        # SQS Daemon sends requests over HTTP - allow and process them before enforcing SSL.
+        app.config.middleware.insert_before(ActionDispatch::SSL, Aws::Rails::EbsSqsActiveJobMiddleware)
+      else
+        app.config.middleware.use(Aws::Rails::EbsSqsActiveJobMiddleware)
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/configuration.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    module SqsActiveJob
+
+      # @return [Configuration] the (singleton) Configuration
+      def self.config
+        @config ||= Configuration.new
+      end
+
+      # @yield Configuration
+      def self.configure
+        yield(config)
+      end
+
+      def self.fifo?(queue_url)
+        queue_url.ends_with? '.fifo'
+      end
+
+      # Configuration for AWS SQS ActiveJob.
+      # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
+      class Configuration
+
+        # Default configuration options
+        # @api private
+        DEFAULTS = {
+          max_messages:  10,
+          visibility_timeout: 120,
+          shutdown_timeout: 15,
+          queues: {},
+          logger: ::Rails.logger,
+          message_group_id: 'SqsActiveJobGroup'
+        }
+
+        # @api private
+        attr_accessor :queues, :max_messages, :visibility_timeout,
+                      :shutdown_timeout, :client, :logger,
+                      :async_queue_error_handler, :message_group_id
+
+        # Don't use this method directly: Confugration is a singleton class, use
+        # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
+        #
+        # @param [Hash] options
+        # @option options [Hash[Symbol, String]] :queues A mapping between the
+        #   active job queue name and the SQS Queue URL. Note: multiple active
+        #   job queues can map to the same SQS Queue URL.
+        #
+        # @option options  [Integer] :max_messages
+        #    The max number of messages to poll for in a batch.
+        #
+        # @option options [Integer] :visibility_timeout
+        #   The visibility timeout is the number of seconds
+        #   that a message will not be processable by any other consumers.
+        #   You should set this value to be longer than your expected job runtime
+        #   to prevent other processes from picking up an running job.
+        #   See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
+        #
+        # @option options [Integer] :shutdown_timeout
+        #   the amount of time to wait
+        #   for a clean shutdown.  Jobs that are unable to complete in this time
+        #   will not be deleted from the SQS queue and will be retryable after
+        #   the visibility timeout.
+        #
+        # @option options [ActiveSupport::Logger] :logger Logger to use
+        #   for the poller.
+        #
+        # @option options [String] :config_file
+        #   Override file to load configuration from.  If not specified will
+        #   attempt to load from config/aws_sqs_active_job.yml.
+        #
+        # @option options [String] :message_group_id (SqsActiveJobGroup)
+        #  The message_group_id to use for queueing messages on a fifo queues.
+        #  Applies only to jobs queued on FIFO queues.
+        #  See the (SQS FIFO Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html]
+        #
+        # @option options [Callable] :async_queue_error_handler An error handler
+        #   to be called when the async active job adapter experiances an error
+        #   queueing a job.  Only applies when
+        #   +active_job.queue_adapter = :amazon_sqs_async+.  Called with:
+        #   [error, job, job_options]
+        #
+        # @option options [SQS::Client] :client SQS Client to use.  A default
+        #   client will be created if none is provided.
+        def initialize(options = {})
+          options[:config_file] ||= config_file if config_file.exist?
+          options = DEFAULTS
+             .merge(file_options(options))
+             .merge(options)
+          set_attributes(options)
+        end
+
+        def client
+          @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
+        end
+
+        # Return the queue_url for a given job_queue name
+        def queue_url_for(job_queue)
+          job_queue = job_queue.to_sym
+          raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
+
+          queues[job_queue.to_sym]
+        end
+
+        # @api private
+        def to_s
+          to_h.to_s
+        end
+
+        # @api private
+        def to_h
+          h = {}
+          self.instance_variables.each do |v|
+            v_sym = v.to_s.gsub('@', '').to_sym
+            val = self.instance_variable_get(v)
+            h[v_sym] = val
+          end
+          h
+        end
+
+        private
+
+        # Set accessible attributes after merged options.
+        def set_attributes(options)
+          options.keys.each do |opt_name|
+            instance_variable_set("@#{opt_name}", options[opt_name])
+          end
+        end
+
+        def file_options(options = {})
+          file_path = config_file_path(options)
+          if file_path
+            load_from_file(file_path)
+          else
+            {}
+          end
+        end
+
+        def config_file
+          file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
+          file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
+          file
+        end
+
+        # Load options from YAML file
+        def load_from_file(file_path)
+          require "erb"
+          opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
+          opts.deep_symbolize_keys
+        end
+
+        # @return [String] Configuration path found in environment or YAML file.
+        def config_file_path(options)
+          options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
+        end
+
+        def user_agent
+          "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
+        end
+      end
+    end
+  end
+end