aws-sdk-rails 3.9.1 → 3.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1491284a363942f9a0d4fe8712449caf2989869377fdb283632bec9c11b71b0b
-  data.tar.gz: c1f65e37b61266b80a20c06c21a39f84f7283ade2b9a590e5b80d3963ada71c7
+  metadata.gz: b6ff9a7f26db95b0ae83526c594bb985b991da455979daf7d2a6ea92be0a5187
+  data.tar.gz: bbb91361034021720e0f43058777b530212d6c43d489f23999b636dd7e187edb
 SHA512:
-  metadata.gz: 456ead7237daf5d83f55f4858f11d40a3d213a3bd8f8ded27c56be10e94e7aba6708a2adad8739b324991f28d22b97e32b8272d073f8f0bc082d390abfff8267
-  data.tar.gz: 97da8e92ec42c8404af945049feae80044aa2d6d1fed276eaf40bd50baa2b07826cfd201ed7eda00a92a9d1636749aaec0cc1adbc159a300c1d5b9690c86fc96
+  metadata.gz: a28a3bf351332f237ef4ce23bbcdb6115ac41468d14dd3d2e1eb76c7a395c33697c9f8c55412fdad3a2b4fc63cbf7dca350243bdf5651743a7ccadfb9adebb0a
+  data.tar.gz: 80076e9047b439cdbd4fe8fe2da651b3cc9d23ed3346fb22c32724b9b3e00463dd77a52f99934bfe26c79ce6d9cc9a17d74fdf6683c9dc5b85e52c40a80f0d9d

data/VERSION

@@ -1 +1 @@
-3.9.1
+3.11.0

data/lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    class AmazonSqsAdapter
+      # == build request parameter of Aws::SQS::Client
+      class Params
+        class << self
+          def assured_delay_seconds(timestamp)
+            delay = (timestamp - Time.now.to_f).floor
+            delay = 0 if delay.negative?
+            raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
+
+            delay
+          end
+        end
+
+        def initialize(job, body)
+          @job = job
+          @body = body || job.serialize
+        end
+
+        def queue_url
+          @queue_url ||= Aws::Rails::SqsActiveJob.config.queue_url_for(@job.queue_name)
+        end
+
+        def entry
+          if Aws::Rails::SqsActiveJob.fifo?(queue_url)
+            default_entry.merge(options_for_fifo)
+          else
+            default_entry
+          end
+        end
+
+        private
+
+        def default_entry
+          {
+            message_body: Aws::Json.dump(@body),
+            message_attributes: message_attributes
+          }
+        end
+
+        def message_attributes
+          {
+            'aws_sqs_active_job_class' => {
+              string_value: @job.class.to_s,
+              data_type: 'String'
+            },
+            'aws_sqs_active_job_version' => {
+              string_value: Aws::Rails::VERSION,
+              data_type: 'String'
+            }
+          }
+        end
+
+        def options_for_fifo
+          options = {}
+          options[:message_deduplication_id] =
+            Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body))
+
+          message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id)
+          message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
+
+          options[:message_group_id] = message_group_id
+          options
+        end
+
+        def deduplication_body
+          ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys)
+          ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
+
+          @body.except(*ex_dedup_keys)
+        end
+      end
+    end
+  end
+end

data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb

@@ -10,55 +10,44 @@ module ActiveJob
       end
 
       def enqueue_at(job, timestamp)
-        delay = (timestamp - Time.now.to_f).floor
-
-        delay = 0 if delay.negative?
-        raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
-
+        delay = Params.assured_delay_seconds(timestamp)
         _enqueue(job, nil, delay_seconds: delay)
       end
 
+      def enqueue_all(jobs)
+        enqueued_count = 0
+        jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs|
+          queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(queue_name)
+          base_send_message_opts = { queue_url: queue_url }
+
+          same_queue_jobs.each_slice(10) do |chunk|
+            entries = chunk.map do |job|
+              entry = Params.new(job, nil).entry
+              entry[:id] = job.job_id
+              entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
+              entry
+            end
+
+            send_message_opts = base_send_message_opts.deep_dup
+            send_message_opts[:entries] = entries
+
+            send_message_batch_result = Aws::Rails::SqsActiveJob.config.client.send_message_batch(send_message_opts)
+            enqueued_count += send_message_batch_result.successful.count
+          end
+        end
+        enqueued_count
+      end
+
       private
 
       def _enqueue(job, body = nil, send_message_opts = {})
         body ||= job.serialize
-        queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
-        send_message_opts[:queue_url] = queue_url
-        send_message_opts[:message_body] = Aws::Json.dump(body)
-        send_message_opts[:message_attributes] = message_attributes(job)
-
-        if Aws::Rails::SqsActiveJob.fifo?(queue_url)
-          send_message_opts[:message_deduplication_id] =
-            Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body(job, body)))
-
-          message_group_id = job.message_group_id if job.respond_to?(:message_group_id)
-          message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
-
-          send_message_opts[:message_group_id] = message_group_id
-        end
+        params = Params.new(job, body)
+        send_message_opts = send_message_opts.merge(params.entry)
+        send_message_opts[:queue_url] = params.queue_url
 
         Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
       end
-
-      def message_attributes(job)
-        {
-          'aws_sqs_active_job_class' => {
-            string_value: job.class.to_s,
-            data_type: 'String'
-          },
-          'aws_sqs_active_job_version' => {
-            string_value: Aws::Rails::VERSION,
-            data_type: 'String'
-          }
-        }
-      end
-
-      def deduplication_body(job, body)
-        ex_dedup_keys = job.excluded_deduplication_keys if job.respond_to?(:excluded_deduplication_keys)
-        ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
-
-        body.except(*ex_dedup_keys)
-      end
     end
 
     # create an alias to allow `:amazon` to be used as the adapter name

data/lib/aws/rails/sqs_active_job/configuration.rb

@@ -25,6 +25,7 @@ module Aws
         DEFAULTS = {
           max_messages: 10,
           shutdown_timeout: 15,
+          retry_standard_errors: true, # TODO: Remove in next MV
           queues: {},
           logger: ::Rails.logger,
           message_group_id: 'SqsActiveJobGroup',
@@ -64,6 +65,16 @@ module Aws
         #   will not be deleted from the SQS queue and will be retryable after
         #   the visibility timeout.
         #
+        # @ option options [Boolean] :retry_standard_errors
+        #   If `true`, StandardErrors raised by ActiveJobs are left on the queue
+        #   and will be retried (pending the SQS Queue's redrive/DLQ/maximum receive settings).
+        #   This behavior overrides the standard Rails ActiveJob
+        #   [Retry/Discard for failed jobs](https://guides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs)
+        #   behavior.  When set to `true` the retries provided by this will be
+        #   on top of any retries configured on the job with `retry_on`.
+        #   When `false`, retry behavior is fully configured
+        #   through `retry_on`/`discard_on` on the ActiveJobs.
+        #
         # @option options [ActiveSupport::Logger] :logger Logger to use
         #   for the poller.
         #
@@ -126,7 +137,7 @@ module Aws
         def to_h
           h = {}
           instance_variables.each do |v|
-            v_sym = v.to_s.gsub('@', '').to_sym
+            v_sym = v.to_s.delete('@').to_sym
             val = instance_variable_get(v)
             h[v_sym] = val
           end

data/lib/aws/rails/sqs_active_job/executor.rb

@@ -13,14 +13,15 @@ module Aws
           auto_terminate: true,
           idletime: 60, # 1 minute
           fallback_policy: :caller_runs # slow down the producer thread
+          # TODO: Consider catching the exception and sleeping instead of using :caller_runs
         }.freeze
 
         def initialize(options = {})
           @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
+          @retry_standard_errors = options[:retry_standard_errors]
           @logger = options[:logger] || ActiveSupport::Logger.new($stdout)
         end
 
-        # TODO: Consider catching the exception and sleeping instead of using :caller_runs
         def execute(message)
           @executor.post(message) do |message|
             begin
@@ -31,10 +32,18 @@ module Aws
             rescue Aws::Json::ParseError => e
               @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
             rescue StandardError => e
-              # message will not be deleted and will be retried
               job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
               @logger.info "Error processing job #{job_msg}: #{e}"
               @logger.debug e.backtrace.join("\n")
+
+              if @retry_standard_errors && !job.exception_executions?
+                @logger.info(
+                  'retry_standard_errors is enabled and job has not ' \
+                  "been retried by Rails.  Leaving #{job_msg} in the queue."
+                )
+              else
+                message.delete
+              end
             end
           end
         end

data/lib/aws/rails/sqs_active_job/job_runner.rb

@@ -15,6 +15,11 @@ module Aws
         def run
           ActiveJob::Base.execute @job_data
         end
+
+        def exception_executions?
+          @job_data['exception_executions'] &&
+            !@job_data['exception_executions'].empty?
+        end
       end
     end
   end

data/lib/aws/rails/sqs_active_job/poller.rb

@@ -16,7 +16,8 @@ module Aws
           threads: 2 * Concurrent.processor_count,
           max_messages: 10,
           shutdown_timeout: 15,
-          backpressure: 10
+          backpressure: 10,
+          retry_standard_errors: true
         }.freeze
 
         def initialize(args = ARGV)
@@ -45,7 +46,12 @@ module Aws
 
           Signal.trap('INT') { raise Interrupt }
           Signal.trap('TERM') { raise Interrupt }
-          @executor = Executor.new(max_threads: @options[:threads], logger: @logger, max_queue: @options[:backpressure])
+          @executor = Executor.new(
+            max_threads: @options[:threads],
+            logger: @logger,
+            max_queue: @options[:backpressure],
+            retry_standard_errors: @options[:retry_standard_errors]
+          )
 
           poll
         rescue Interrupt
@@ -99,6 +105,7 @@ module Aws
           require File.expand_path('config/environment.rb')
         end
 
+        # rubocop:disable Metrics
         def parse_args(argv)
           out = {}
           parser = ::OptionParser.new do |opts|
@@ -127,6 +134,10 @@ module Aws
                     'The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.') do |a|
               out[:shutdown_timeout] = a
             end
+            opts.on('--[no-]retry_standard_errors [FLAG]', TrueClass,
+                    'When set, retry all StandardErrors (leaving failed messages on the SQS Queue). These retries are ON TOP of standard Rails ActiveJob retries set by retry_on in the ActiveJob.') do |a|
+              out[:retry_standard_errors] = a.nil? ? true : a
+            end
           end
 
           parser.banner = 'aws_sqs_active_job [options]'
@@ -138,6 +149,7 @@ module Aws
           parser.parse(argv)
           out
         end
+        # rubocop:enable Metrics
 
         def validate_config
           raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]

data/lib/aws-sdk-rails.rb

@@ -13,6 +13,7 @@ require_relative 'aws/rails/middleware/ebs_sqs_active_job_middleware'
 
 require_relative 'action_dispatch/session/dynamodb_store'
 require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
+require_relative 'active_job/queue_adapters/amazon_sqs_adapter/params'
 require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
 
 require_relative 'generators/aws_record/base'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-rails
 version: !ruby/object:Gem::Version
-  version: 3.9.1
+  version: 3.11.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-19 00:00:00.000000000 Z
+date: 2024-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-record
@@ -152,6 +152,7 @@ files:
 - bin/aws_sqs_active_job
 - lib/action_dispatch/session/dynamodb_store.rb
 - lib/active_job/queue_adapters/amazon_sqs_adapter.rb
+- lib/active_job/queue_adapters/amazon_sqs_adapter/params.rb
 - lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb
 - lib/aws-sdk-rails.rb
 - lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb
@@ -197,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.22
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: AWS SDK for Ruby on Rails Plugin