aws-sdk-rails 3.6.1 → 3.13.0

data/lib/aws/rails/sqs_active_job/deduplication.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    # SQS ActiveJob modules
+    module SqsActiveJob
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :excluded_deduplication_keys
+      end
+
+      # class methods for SQS ActiveJob.
+      module ClassMethods
+        def deduplicate_without(*keys)
+          self.excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/executor.rb

@@ -7,49 +7,68 @@ module Aws
     module SqsActiveJob
       # CLI runner for polling for SQS ActiveJobs
       class Executor
-
         DEFAULTS = {
-           min_threads:     0,
-           max_threads:     Concurrent.processor_count,
-           auto_terminate:  true,
-           idletime:        60, # 1 minute
-           fallback_policy: :caller_runs # slow down the producer thread
+          min_threads: 0,
+          max_threads: Integer(Concurrent.available_processor_count || Concurrent.processor_count),
+          auto_terminate: true,
+          idletime: 60, # 1 minute
+          fallback_policy: :abort # Concurrent::RejectedExecutionError must be handled
         }.freeze
 
         def initialize(options = {})
           @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
-          @logger = options[:logger] || ActiveSupport::Logger.new(STDOUT)
+          @retry_standard_errors = options[:retry_standard_errors]
+          @logger = options[:logger] || ActiveSupport::Logger.new($stdout)
+          @task_complete = Concurrent::Event.new
         end
 
-        # TODO: Consider catching the exception and sleeping instead of using :caller_runs
         def execute(message)
-          @executor.post(message) do |message|
-            begin
-              job = JobRunner.new(message)
-              @logger.info("Running job: #{job.id}[#{job.class_name}]")
-              job.run
-              message.delete
-            rescue Aws::Json::ParseError => e
-              @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
-            rescue StandardError => e
-              # message will not be deleted and will be retried
-              job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
-              @logger.info "Error processing job #{job_msg}: #{e}"
-              @logger.debug e.backtrace.join("\n")
-            end
-          end
+          post_task(message)
+        rescue Concurrent::RejectedExecutionError
+          # no capacity, wait for a task to complete
+          @task_complete.reset
+          @task_complete.wait
+          retry
         end
 
-        def shutdown(timeout=nil)
+        def shutdown(timeout = nil)
           @executor.shutdown
           clean_shutdown = @executor.wait_for_termination(timeout)
           if clean_shutdown
             @logger.info 'Clean shutdown complete.  All executing jobs finished.'
           else
-            @logger.info "Timeout (#{timeout}) exceeded.  Some jobs may not have"\
-              " finished cleanly.  Unfinished jobs will not be removed from"\
-              " the queue and can be ru-run once their visibility timeout"\
-              " passes."
+            @logger.info "Timeout (#{timeout}) exceeded.  Some jobs may not have " \
+                         'finished cleanly.  Unfinished jobs will not be removed from ' \
+                         'the queue and can be ru-run once their visibility timeout ' \
+                         'passes.'
+          end
+        end
+
+        private
+
+        def post_task(message)
+          @executor.post(message) do |message|
+            job = JobRunner.new(message)
+            @logger.info("Running job: #{job.id}[#{job.class_name}]")
+            job.run
+            message.delete
+          rescue Aws::Json::ParseError => e
+            @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
+          rescue StandardError => e
+            job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
+            @logger.info "Error processing job #{job_msg}: #{e}"
+            @logger.debug e.backtrace.join("\n")
+
+            if @retry_standard_errors && !job.exception_executions?
+              @logger.info(
+                'retry_standard_errors is enabled and job has not ' \
+                "been retried by Rails.  Leaving #{job_msg} in the queue."
+              )
+            else
+              message.delete
+            end
+          ensure
+            @task_complete.set
           end
         end
       end

data/lib/aws/rails/sqs_active_job/job_runner.rb

@@ -3,7 +3,6 @@
 module Aws
   module Rails
     module SqsActiveJob
-
       class JobRunner
         attr_reader :id, :class_name
 
@@ -16,6 +15,11 @@ module Aws
         def run
           ActiveJob::Base.execute @job_data
         end
+
+        def exception_executions?
+          @job_data['exception_executions'] &&
+            !@job_data['exception_executions'].empty?
+        end
       end
     end
   end

data/lib/aws/rails/sqs_active_job/lambda_handler.rb

@@ -5,7 +5,6 @@ require 'aws-sdk-sqs'
 module Aws
   module Rails
     module SqsActiveJob
-
       # A lambda event handler to run jobs from an SQS queue trigger
       # Trigger the lambda from your SQS queue
       # Configure the entrypoint to: +config/environment.Aws::Rails::SqsActiveJob.lambda_job_handler+
@@ -23,13 +22,11 @@ module Aws
         "Processed #{event['Records'].length} jobs."
       end
 
-      private
-
       def self.to_sqs_msg(record)
         msg = Aws::SQS::Types::Message.new(
           body: record['body'],
           md5_of_body: record['md5OfBody'],
-          message_attributes: self.to_message_attributes(record),
+          message_attributes: to_message_attributes(record),
           message_id: record['messageId'],
           receipt_handle: record['receiptHandle']
         )
@@ -42,8 +39,8 @@ module Aws
       end
 
       def self.to_message_attributes(record)
-        record['messageAttributes'].each_with_object({}) do |(key, value), acc|
-          acc[key] = {
+        record['messageAttributes'].transform_values do |value|
+          {
             string_value: value['stringValue'],
             binary_value: value['binaryValue'],
             string_list_values: ['stringListValues'],

data/lib/aws/rails/sqs_active_job/poller.rb

@@ -7,20 +7,18 @@ require 'concurrent'
 module Aws
   module Rails
     module SqsActiveJob
-
-      class Interrupt < Exception; end
+      class Interrupt < StandardError; end
 
       # CLI runner for polling for SQS ActiveJobs
       # Use `aws_sqs_active_job --help` for detailed usage
       class Poller
-
         DEFAULT_OPTS = {
-          threads: 2*Concurrent.processor_count,
+          threads: 2 * Concurrent.processor_count,
           max_messages: 10,
-          visibility_timeout: 60,
           shutdown_timeout: 15,
-          backpressure: 10
-        }
+          backpressure: 10,
+          retry_standard_errors: true
+        }.freeze
 
         def initialize(args = ARGV)
           @options = parse_args(args)
@@ -29,7 +27,7 @@ module Aws
         end
 
         def set_environment
-          @environment = @options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+          @environment = @options[:environment] || ENV['APP_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
         end
 
         def run
@@ -43,13 +41,17 @@ module Aws
                      .merge(@options.to_h)
           validate_config
           # ensure we have a logger configured
-          @logger = @options[:logger] || ActiveSupport::Logger.new(STDOUT)
+          @logger = @options[:logger] || ActiveSupport::Logger.new($stdout)
           @logger.info("Starting Poller with options=#{@options}")
 
-
           Signal.trap('INT') { raise Interrupt }
           Signal.trap('TERM') { raise Interrupt }
-          @executor = Executor.new(max_threads: @options[:threads], logger: @logger, max_queue: @options[:backpressure])
+          @executor = Executor.new(
+            max_threads: @options[:threads],
+            logger: @logger,
+            max_queue: @options[:backpressure],
+            retry_standard_errors: @options[:retry_standard_errors]
+          )
 
           poll
         rescue Interrupt
@@ -79,9 +81,7 @@ module Aws
           # in order
           # Jobs with different message_group_id will be processed in
           # parallel and may be out of order.
-          if Aws::Rails::SqsActiveJob.fifo?(queue_url)
-            poller_options[:max_number_of_messages] = 1
-          end
+          poller_options[:max_number_of_messages] = 1 if Aws::Rails::SqsActiveJob.fifo?(queue_url)
 
           single_message = poller_options[:max_number_of_messages] == 1
 
@@ -90,35 +90,58 @@ module Aws
             @logger.info "Processing batch of #{msgs.length} messages"
             msgs.each do |msg|
               @executor.execute(Aws::SQS::Message.new(
-                queue_url: queue_url,
-                receipt_handle: msg.receipt_handle,
-                data: msg,
-                client: client
-              ))
+                                  queue_url: queue_url,
+                                  receipt_handle: msg.receipt_handle,
+                                  data: msg,
+                                  client: client
+                                ))
             end
           end
         end
 
         def boot_rails
           ENV['RACK_ENV'] = ENV['RAILS_ENV'] = @environment
-          require "rails"
-          require File.expand_path("config/environment.rb")
+          require 'rails'
+          require File.expand_path('config/environment.rb')
         end
 
+        # rubocop:disable Metrics
         def parse_args(argv)
           out = {}
-          parser = ::OptionParser.new { |opts|
-            opts.on("-q", "--queue STRING", "[Required] Queue to poll") { |a| out[:queue] = a }
-            opts.on("-e", "--environment STRING", "Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.") { |a| out[:environment] = a }
-            opts.on("-t", "--threads INTEGER", Integer, "The maximum number of worker threads to create.  Defaults to 2x the number of processors available on this system.") { |a| out[:threads] = a }
-            opts.on("-b", "--backpressure INTEGER", Integer, "The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.") { |a| out[:backpressure] = a }
-            opts.on("-m", "--max_messages INTEGER", Integer, "Max number of messages to receive in a batch from SQS.") { |a| out[:max_messages] = a }
-            opts.on("-v", "--visibility_timeout INTEGER", Integer, "The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.") { |a| out[:visibility_timeout] = a }
-            opts.on("-s", "--shutdown_timeout INTEGER", Integer, "The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.") { |a| out[:shutdown_timeout] = a }
-          }
+          parser = ::OptionParser.new do |opts|
+            opts.on('-q', '--queue STRING', '[Required] Queue to poll') { |a| out[:queue] = a }
+            opts.on('-e', '--environment STRING',
+                    'Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.') do |a|
+              out[:environment] = a
+            end
+            opts.on('-t', '--threads INTEGER', Integer,
+                    'The maximum number of worker threads to create.  Defaults to 2x the number of processors available on this system.') do |a|
+              out[:threads] = a
+            end
+            opts.on('-b', '--backpressure INTEGER', Integer,
+                    'The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.') do |a|
+              out[:backpressure] = a
+            end
+            opts.on('-m', '--max_messages INTEGER', Integer,
+                    'Max number of messages to receive in a batch from SQS.') do |a|
+              out[:max_messages] = a
+            end
+            opts.on('-v', '--visibility_timeout INTEGER', Integer,
+                    'The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.') do |a|
+              out[:visibility_timeout] = a
+            end
+            opts.on('-s', '--shutdown_timeout INTEGER', Integer,
+                    'The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.') do |a|
+              out[:shutdown_timeout] = a
+            end
+            opts.on('--[no-]retry_standard_errors [FLAG]', TrueClass,
+                    'When set, retry all StandardErrors (leaving failed messages on the SQS Queue). These retries are ON TOP of standard Rails ActiveJob retries set by retry_on in the ActiveJob.') do |a|
+              out[:retry_standard_errors] = a.nil? ? true : a
+            end
+          end
 
-          parser.banner = "aws_sqs_active_job [options]"
-          parser.on_tail "-h", "--help", "Show help" do
+          parser.banner = 'aws_sqs_active_job [options]'
+          parser.on_tail '-h', '--help', 'Show help' do
             puts parser
             exit 1
           end
@@ -126,6 +149,7 @@ module Aws
           parser.parse(argv)
           out
         end
+        # rubocop:enable Metrics
 
         def validate_config
           raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]

data/lib/aws-sdk-rails.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
-require_relative 'aws/rails/mailer'
+require_relative 'aws/rails/ses_mailer'
+require_relative 'aws/rails/sesv2_mailer'
 require_relative 'aws/rails/railtie'
 require_relative 'aws/rails/notifications'
 require_relative 'aws/rails/sqs_active_job/configuration'
+require_relative 'aws/rails/sqs_active_job/deduplication'
 require_relative 'aws/rails/sqs_active_job/executor'
 require_relative 'aws/rails/sqs_active_job/job_runner'
 require_relative 'aws/rails/sqs_active_job/lambda_handler'
@@ -11,6 +13,7 @@ require_relative 'aws/rails/middleware/ebs_sqs_active_job_middleware'
 
 require_relative 'action_dispatch/session/dynamodb_store'
 require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
+require_relative 'active_job/queue_adapters/amazon_sqs_adapter/params'
 require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
 
 require_relative 'generators/aws_record/base'