aws-sdk-rails 3.2.1 → 3.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b16c1ca75c59341fdc3696d91648fda6901937f31b5b4c55b5d1ccf3bcc9af47
-  data.tar.gz: 9c9c22d4740ea76b06fa7ee6b8d525450fc982e3c73db0fe81234356dbd71a86
+  metadata.gz: 839c723ba30e8acd11e135351278a0422b4f7e6a09a364b8a3fda7d33aa2e899
+  data.tar.gz: af1ba6b6525f17e6a46cc16aa4ccbaaf8317860ddf5cd6128d3883c82660ee24
 SHA512:
-  metadata.gz: 2ef9e3bf5bde8d91b25635e2bed4028a163547f6e8eabb3ece2e08d1482e6edce16386a916240f344dcce27bc8efec3c5c424236fed6f150e4a6912165648914
-  data.tar.gz: 0052b3633ce5fc655f4b6fdd80851049196e1c7c7a7d6cc92e17d52fc6fcc8ae69712bc411d3f0feb369dfd835e23912c67802ff32abe3654cf3b4feee87c02a
+  metadata.gz: 8288a0c8551bc30cac782c7510a825fdcd9d8ba31bd1d2feda8c2a20ad29f569194237f686f191f1a57ee12eb91a08bb1c5b65b07519af5b62bd7f56ab3f79cb
+  data.tar.gz: d6835fc12e6a4a5bc9573b4c5b997a1df5d17e49c28ea2ea16e76f8ebc7b2c8c515026b0a48fb0b15eaab3ba22f1904782f553a03a784dcd665d71723ddd47bd

data/VERSION

@@ -0,0 +1 @@
+3.6.1

data/bin/aws_sqs_active_job

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/aws/rails/sqs_active_job/poller'
+
+Aws::Rails::SqsActiveJob::Poller.new.run

data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+module ActiveJob
+  module QueueAdapters
+
+    class AmazonSqsAdapter
+
+      def enqueue(job)
+        _enqueue(job)
+      end
+
+      def enqueue_at(job, timestamp, opts={})
+        delay = (timestamp - Time.now.to_f).floor
+        raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
+        _enqueue(job, delay_seconds: delay)
+      end
+
+      private
+
+      def _enqueue(job, send_message_opts = {})
+        body = job.serialize
+        queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
+        send_message_opts[:queue_url] = queue_url
+        send_message_opts[:message_body] = Aws::Json.dump(body)
+        send_message_opts[:message_attributes] = message_attributes(job)
+
+        if Aws::Rails::SqsActiveJob.fifo?(queue_url)
+          # job_id is unique per initialization of job
+          # Remove it from message dup id to ensure run-once behavior
+          # with ActiveJob retries
+           send_message_opts[:message_deduplication_id] =
+             Digest::SHA256.hexdigest(
+               Aws::Json.dump(body.except('job_id'))
+             )
+
+           send_message_opts[:message_group_id] = Aws::Rails::SqsActiveJob.config.message_group_id
+        end
+        Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
+      end
+
+      def message_attributes(job)
+        {
+          'aws_sqs_active_job_class' => {
+            string_value: job.class.to_s,
+            data_type: 'String'
+          },
+          'aws_sqs_active_job_version' => {
+            string_value: Aws::Rails::VERSION,
+            data_type: 'String'
+          }
+        }
+      end
+    end
+
+    # create an alias to allow `:amazon` to be used as the adapter name
+    # `:amazon` is the convention used for ActionMailer and ActiveStorage
+    AmazonAdapter = AmazonSqsAdapter
+  end
+end

data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'concurrent'
+
+module ActiveJob
+  module QueueAdapters
+
+    # == Async adapter for Amazon SQS ActiveJob
+    #
+    # This adapter queues jobs asynchronously (ie non-blocking).  Error handler can be configured
+    # with +Aws::Rails::SqsActiveJob.config.async_queue_error_handler+.
+    #
+    # To use this adapter, set up as:
+    #
+    # config.active_job.queue_adapter = :amazon_sqs_async
+    class AmazonSqsAsyncAdapter < AmazonSqsAdapter
+
+      private
+
+      def _enqueue(job, send_message_opts = {})
+        # FIFO jobs must be queued in order, so do not queue async
+        queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
+        if Aws::Rails::SqsActiveJob.fifo?(queue_url)
+          super(job, send_message_opts)
+        else
+          Concurrent::Promise
+          .execute { super(job, send_message_opts) }
+          .on_error do |e|
+            Rails.logger.error "Failed to queue job #{job}.  Reason: #{e}"
+            error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
+            error_handler.call(e, job, send_message_opts) if error_handler
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-rails.rb

@@ -3,5 +3,20 @@
 require_relative 'aws/rails/mailer'
 require_relative 'aws/rails/railtie'
 require_relative 'aws/rails/notifications'
+require_relative 'aws/rails/sqs_active_job/configuration'
+require_relative 'aws/rails/sqs_active_job/executor'
+require_relative 'aws/rails/sqs_active_job/job_runner'
+require_relative 'aws/rails/sqs_active_job/lambda_handler'
+require_relative 'aws/rails/middleware/ebs_sqs_active_job_middleware'
 
 require_relative 'action_dispatch/session/dynamodb_store'
+require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
+require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
+
+require_relative 'generators/aws_record/base'
+
+module Aws
+  module Rails
+    VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
+  end
+end

data/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    # Middleware to handle requests from the SQS Daemon present on Elastic Beanstalk worker environments.
+    class EbsSqsActiveJobMiddleware
+      INTERNAL_ERROR_MESSAGE = 'Failed to execute job - see Rails log for more details.'
+      INTERNAL_ERROR_RESPONSE = [500, { 'Content-Type' => 'text/plain' }, [INTERNAL_ERROR_MESSAGE]].freeze
+      FORBIDDEN_MESSAGE = 'Request with aws-sqsd user agent was made from untrusted address.'
+      FORBIDDEN_RESPONSE = [403, { 'Content-Type' => 'text/plain' }, [FORBIDDEN_MESSAGE]].freeze
+
+      def initialize(app)
+        @app = app
+        @logger = ActiveSupport::Logger.new(STDOUT)
+      end
+
+      def call(env)
+        request = ActionDispatch::Request.new(env)
+
+        # Pass through unless user agent is the SQS Daemon
+        return @app.call(env) unless from_sqs_daemon?(request)
+
+        @logger.debug('aws-rails-sdk middleware detected call from Elastic Beanstalk SQS Daemon.')
+
+        # Only accept requests from this user agent if it is from localhost or a docker host in case of forgery.
+        unless request.local? || sent_from_docker_host?(request)
+          @logger.warn("SQSD request detected from untrusted address #{request.remote_ip}; returning 403 forbidden.")
+          return FORBIDDEN_RESPONSE
+        end
+
+        # Execute job or periodic task based on HTTP request context
+        periodic_task?(request) ? execute_periodic_task(request) : execute_job(request)
+      end
+
+      private
+
+      def execute_job(request)
+        # Jobs queued from the Active Job SQS adapter contain the JSON message in the request body.
+        job = Aws::Json.load(request.body.string)
+        job_name = job['job_class']
+        @logger.debug("Executing job: #{job_name}")
+
+        begin
+          ActiveJob::Base.execute(job)
+        rescue NoMethodError, NameError => e
+          @logger.error("Job #{job_name} could not resolve to a class that inherits from Active Job.")
+          @logger.error("Error: #{e}")
+          return INTERNAL_ERROR_RESPONSE
+        end
+
+        [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran job #{job_name}."]]
+      end
+
+      def execute_periodic_task(request)
+        # The beanstalk worker SQS Daemon will add the 'X-Aws-Sqsd-Taskname' for periodic tasks set in cron.yaml.
+        job_name = request.headers['X-Aws-Sqsd-Taskname']
+        @logger.debug("Creating and executing periodic task: #{job_name}")
+
+        begin
+          job = job_name.constantize.new
+          job.perform_now
+        rescue NoMethodError, NameError => e
+          @logger.error("Periodic task #{job_name} could not resolve to an Active Job class - check the spelling in cron.yaml.")
+          @logger.error("Error: #{e}.")
+          return INTERNAL_ERROR_RESPONSE
+        end
+
+        [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran periodic task #{job_name}."]]
+      end
+
+      # The beanstalk worker SQS Daemon sets a specific User-Agent headers that begins with 'aws-sqsd'.
+      def from_sqs_daemon?(request)
+        current_user_agent = request.headers['User-Agent']
+
+        !current_user_agent.nil? && current_user_agent.start_with?('aws-sqsd')
+      end
+
+      # The beanstalk worker SQS Daemon will add the custom 'X-Aws-Sqsd-Taskname' header for periodic tasks set in cron.yaml.
+      def periodic_task?(request)
+        !request.headers['X-Aws-Sqsd-Taskname'].nil? && request.headers['X-Aws-Sqsd-Taskname'].present?
+      end
+
+      def sent_from_docker_host?(request)
+        app_runs_in_docker_container? && request.remote_ip == '172.17.0.1'
+      end
+
+      def app_runs_in_docker_container?
+        @app_runs_in_docker_container ||= `[ -f /proc/1/cgroup ] && cat /proc/1/cgroup` =~ /docker/
+      end
+    end
+  end
+end

data/lib/aws/rails/railtie.rb

@@ -13,8 +13,13 @@ module Aws
         Aws::Rails.log_to_rails_logger
       end
 
+      initializer 'aws-sdk-rails.insert_middleware' do |app|
+        Aws::Rails.add_sqsd_middleware(app)
+      end
+
       rake_tasks do
         load 'tasks/dynamo_db/session_store.rake'
+        load 'tasks/aws_record/migrate.rake'
       end
     end
 
@@ -43,10 +48,7 @@ module Aws
       aws_credential_keys = %i[access_key_id secret_access_key session_token]
 
       Aws.config.merge!(
-        ::Rails.application
-          .try(:credentials)
-          .try(:aws)
-          .to_h.slice(*aws_credential_keys)
+        ::Rails.application.credentials[:aws].to_h.slice(*aws_credential_keys)
       )
     end
 
@@ -63,5 +65,21 @@ module Aws
         end
       end
     end
+
+    # Register a middleware that will handle requests from the Elastic Beanstalk worker SQS Daemon.
+    # This will only be added in the presence of the AWS_PROCESS_BEANSTALK_WORKER_REQUESTS environment variable.
+    # The expectation is this variable should only be set on EB worker environments.
+    def self.add_sqsd_middleware(app)
+      is_eb_worker_hosted = Aws::Util.str_2_bool(ENV['AWS_PROCESS_BEANSTALK_WORKER_REQUESTS'].to_s.downcase)
+
+      return unless is_eb_worker_hosted
+
+      if app.config.force_ssl
+        # SQS Daemon sends requests over HTTP - allow and process them before enforcing SSL.
+        app.config.middleware.insert_before(ActionDispatch::SSL, Aws::Rails::EbsSqsActiveJobMiddleware)
+      else
+        app.config.middleware.use(Aws::Rails::EbsSqsActiveJobMiddleware)
+      end
+    end
   end
 end

data/lib/aws/rails/sqs_active_job/configuration.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    module SqsActiveJob
+
+      # @return [Configuration] the (singleton) Configuration
+      def self.config
+        @config ||= Configuration.new
+      end
+
+      # @yield Configuration
+      def self.configure
+        yield(config)
+      end
+
+      def self.fifo?(queue_url)
+        queue_url.ends_with? '.fifo'
+      end
+
+      # Configuration for AWS SQS ActiveJob.
+      # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
+      class Configuration
+
+        # Default configuration options
+        # @api private
+        DEFAULTS = {
+          max_messages:  10,
+          visibility_timeout: 120,
+          shutdown_timeout: 15,
+          queues: {},
+          logger: ::Rails.logger,
+          message_group_id: 'SqsActiveJobGroup'
+        }
+
+        # @api private
+        attr_accessor :queues, :max_messages, :visibility_timeout,
+                      :shutdown_timeout, :client, :logger,
+                      :async_queue_error_handler, :message_group_id
+
+        # Don't use this method directly: Confugration is a singleton class, use
+        # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
+        #
+        # @param [Hash] options
+        # @option options [Hash[Symbol, String]] :queues A mapping between the
+        #   active job queue name and the SQS Queue URL. Note: multiple active
+        #   job queues can map to the same SQS Queue URL.
+        #
+        # @option options  [Integer] :max_messages
+        #    The max number of messages to poll for in a batch.
+        #
+        # @option options [Integer] :visibility_timeout
+        #   The visibility timeout is the number of seconds
+        #   that a message will not be processable by any other consumers.
+        #   You should set this value to be longer than your expected job runtime
+        #   to prevent other processes from picking up an running job.
+        #   See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
+        #
+        # @option options [Integer] :shutdown_timeout
+        #   the amount of time to wait
+        #   for a clean shutdown.  Jobs that are unable to complete in this time
+        #   will not be deleted from the SQS queue and will be retryable after
+        #   the visibility timeout.
+        #
+        # @option options [ActiveSupport::Logger] :logger Logger to use
+        #   for the poller.
+        #
+        # @option options [String] :config_file
+        #   Override file to load configuration from.  If not specified will
+        #   attempt to load from config/aws_sqs_active_job.yml.
+        #
+        # @option options [String] :message_group_id (SqsActiveJobGroup)
+        #  The message_group_id to use for queueing messages on a fifo queues.
+        #  Applies only to jobs queued on FIFO queues.
+        #  See the (SQS FIFO Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html]
+        #
+        # @option options [Callable] :async_queue_error_handler An error handler
+        #   to be called when the async active job adapter experiances an error
+        #   queueing a job.  Only applies when
+        #   +active_job.queue_adapter = :amazon_sqs_async+.  Called with:
+        #   [error, job, job_options]
+        #
+        # @option options [SQS::Client] :client SQS Client to use.  A default
+        #   client will be created if none is provided.
+        def initialize(options = {})
+          options[:config_file] ||= config_file if config_file.exist?
+          options = DEFAULTS
+             .merge(file_options(options))
+             .merge(options)
+          set_attributes(options)
+        end
+
+        def client
+          @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
+        end
+
+        # Return the queue_url for a given job_queue name
+        def queue_url_for(job_queue)
+          job_queue = job_queue.to_sym
+          raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
+
+          queues[job_queue.to_sym]
+        end
+
+        # @api private
+        def to_s
+          to_h.to_s
+        end
+
+        # @api private
+        def to_h
+          h = {}
+          self.instance_variables.each do |v|
+            v_sym = v.to_s.gsub('@', '').to_sym
+            val = self.instance_variable_get(v)
+            h[v_sym] = val
+          end
+          h
+        end
+
+        private
+
+        # Set accessible attributes after merged options.
+        def set_attributes(options)
+          options.keys.each do |opt_name|
+            instance_variable_set("@#{opt_name}", options[opt_name])
+          end
+        end
+
+        def file_options(options = {})
+          file_path = config_file_path(options)
+          if file_path
+            load_from_file(file_path)
+          else
+            {}
+          end
+        end
+
+        def config_file
+          file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
+          file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
+          file
+        end
+
+        # Load options from YAML file
+        def load_from_file(file_path)
+          require "erb"
+          opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
+          opts.deep_symbolize_keys
+        end
+
+        # @return [String] Configuration path found in environment or YAML file.
+        def config_file_path(options)
+          options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
+        end
+
+        def user_agent
+          "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
+        end
+      end
+    end
+  end
+end