aws-sdk-rails 3.7.1 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48698b21d8248ac597ad8fd9d334e52d9399cf5880c994ffe556aa2e988f81e6
-  data.tar.gz: bd88191e84e5676435a2d2135ad6f6835cb8c624eef573c688a81c3bd9e8f255
+  metadata.gz: 79de6d1c1c8319fd70a36949eebb7630630bbd9b8a566b6f473d51590e48af38
+  data.tar.gz: a7d383cfecc470333c998f61ca380330dac77dc13dc5d302a5ba6bb34f8e5a81
 SHA512:
-  metadata.gz: ec0f7a6f90b736285486d3a36d7aab50cf1d368d9ea69f4b76031634b16df1e899d8146586b9b806f0921f2afebecc5444fe25badf4f0d8bf54a3cf42f472b3c
-  data.tar.gz: bbfe9b2488cf8c141a5e196a2ebe9631705ed176327db28a1503213f82aac81f3ec37be4f77109665fbf3690c344c52870b39f5c12f35d031b89387b057bbe5a
+  metadata.gz: 95e0b8f6e3e565b6aeba24955bdbacff0a280de8c7f8a976a75ebadfd28710fc47c34c4fc079d95bb7d2678f60c185ec158719864eee0f8935424d4adb99d2e1
+  data.tar.gz: 95c7da0d1ac045139de0e1b4f251b1a118cadfa62a4eef3bfe8925df0e914325f7523fb3009cc3bb433828d4ea916f0c1e12b919a689931773317492a6811156

data/VERSION

@@ -1 +1 @@
-3.7.1
+3.9.0

data/bin/aws_sqs_active_job

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require_relative '../lib/aws/rails/sqs_active_job/poller'
 

data/lib/action_dispatch/session/dynamodb_store.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'aws-sessionstore-dynamodb'
 require 'action_dispatch/middleware/session/abstract_store'
 
@@ -19,7 +21,7 @@ module ActionDispatch
       include SessionObject
 
       def initialize(app, options = {})
-        options[:config_file] ||= config_file if config_file.exist?
+        options[:config_file] ||= config_file if File.exist?(config_file)
         options[:secret_key] ||= Rails.application.secret_key_base
         super
       end
@@ -28,7 +30,7 @@ module ActionDispatch
 
       def config_file
         file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
-        file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
+        file = Rails.root.join('config/dynamo_db_session_store.yml') unless File.exist?(file)
         file
       end
     end

data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb

@@ -4,9 +4,7 @@ require 'aws-sdk-sqs'
 
 module ActiveJob
   module QueueAdapters
-
     class AmazonSqsAdapter
-
       def enqueue(job)
         _enqueue(job)
       end
@@ -14,6 +12,7 @@ module ActiveJob
       def enqueue_at(job, timestamp)
         delay = (timestamp - Time.now.to_f).floor
         raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
+
         _enqueue(job, nil, delay_seconds: delay)
       end
 
@@ -27,11 +26,8 @@ module ActiveJob
         send_message_opts[:message_attributes] = message_attributes(job)
 
         if Aws::Rails::SqsActiveJob.fifo?(queue_url)
-          # job_id is unique per initialization of job
-          # Remove it from message dup id to ensure run-once behavior
-          # with ActiveJob retries
           send_message_opts[:message_deduplication_id] =
-            Digest::SHA256.hexdigest(Aws::Json.dump(body.except('job_id')))
+            Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body(job, body)))
 
           message_group_id = job.message_group_id if job.respond_to?(:message_group_id)
           message_group_id ||= Aws::Rails::SqsActiveJob.config.message_group_id
@@ -54,6 +50,13 @@ module ActiveJob
           }
         }
       end
+
+      def deduplication_body(job, body)
+        ex_dedup_keys = job.excluded_deduplication_keys if job.respond_to?(:excluded_deduplication_keys)
+        ex_dedup_keys ||= Aws::Rails::SqsActiveJob.config.excluded_deduplication_keys
+
+        body.except(*ex_dedup_keys)
+      end
     end
 
     # create an alias to allow `:amazon` to be used as the adapter name

data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb

@@ -5,7 +5,6 @@ require 'concurrent'
 
 module ActiveJob
   module QueueAdapters
-
     # == Async adapter for Amazon SQS ActiveJob
     #
     # This adapter queues jobs asynchronously (ie non-blocking).  Error handler can be configured
@@ -15,7 +14,6 @@ module ActiveJob
     #
     # config.active_job.queue_adapter = :amazon_sqs_async
     class AmazonSqsAsyncAdapter < AmazonSqsAdapter
-
       private
 
       def _enqueue(job, body = nil, send_message_opts = {})

data/lib/aws/rails/notifications.rb

@@ -5,14 +5,12 @@ require 'active_support/notifications'
 
 module Aws
   module Rails
-
     # Instruments client operation calls for ActiveSupport::Notifications
     # Each client operation will produce an event with name:
     # <operation>.<service>.aws
     # @api private
     class Notifications < Seahorse::Client::Plugin
-
-      def add_handlers(handlers, config)
+      def add_handlers(handlers, _config)
         # This plugin needs to be first
         # which means it is called first in the stack, to start recording time,
         # and returns last
@@ -20,7 +18,6 @@ module Aws
       end
 
       class Handler < Seahorse::Client::Handler
-
         def call(context)
           event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
           ActiveSupport::Notifications.instrument(event_name, context: context) do

data/lib/aws/rails/ses_mailer.rb

@@ -20,6 +20,7 @@ module Aws
       #   [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
       def initialize(options = {})
         @client = SES::Client.new(options)
+        @client.config.user_agent_frameworks << 'aws-sdk-rails'
       end
 
       # Rails expects this method to exist, and to handle a Mail::Message object

data/lib/aws/rails/sesv2_mailer.rb

@@ -20,6 +20,7 @@ module Aws
       #   [Aws::SESV2::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SESV2/Client.html#initialize-instance_method).
       def initialize(options = {})
         @client = SESV2::Client.new(options)
+        @client.config.user_agent_frameworks << 'aws-sdk-rails'
       end
 
       # Rails expects this method to exist, and to handle a Mail::Message object

data/lib/aws/rails/sqs_active_job/configuration.rb

@@ -3,7 +3,6 @@
 module Aws
   module Rails
     module SqsActiveJob
-
       # @return [Configuration] the (singleton) Configuration
       def self.config
         @config ||= Configuration.new
@@ -21,23 +20,25 @@ module Aws
       # Configuration for AWS SQS ActiveJob.
       # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
       class Configuration
-
         # Default configuration options
         # @api private
         DEFAULTS = {
-          max_messages:  10,
+          max_messages: 10,
           shutdown_timeout: 15,
           queues: {},
           logger: ::Rails.logger,
-          message_group_id: 'SqsActiveJobGroup'
-        }
+          message_group_id: 'SqsActiveJobGroup',
+          excluded_deduplication_keys: ['job_id']
+        }.freeze
 
         # @api private
         attr_accessor :queues, :max_messages, :visibility_timeout,
                       :shutdown_timeout, :client, :logger,
                       :async_queue_error_handler, :message_group_id
 
-        # Don't use this method directly: Confugration is a singleton class, use
+        attr_reader :excluded_deduplication_keys
+
+        # Don't use this method directly: Configuration is a singleton class, use
         # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
         #
         # @param [Hash] options
@@ -67,7 +68,7 @@ module Aws
         #   for the poller.
         #
         # @option options [String] :config_file
-        #   Override file to load configuration from.  If not specified will
+        #   Override file to load configuration from. If not specified will
         #   attempt to load from config/aws_sqs_active_job.yml.
         #
         # @option options [String] :message_group_id (SqsActiveJobGroup)
@@ -81,18 +82,31 @@ module Aws
         #   +active_job.queue_adapter = :amazon_sqs_async+.  Called with:
         #   [error, job, job_options]
         #
-        # @option options [SQS::Client] :client SQS Client to use.  A default
+        # @option options [SQS::Client] :client SQS Client to use. A default
         #   client will be created if none is provided.
+        #
+        # @option options [Array] :excluded_deduplication_keys (['job_id'])
+        #   The type of keys stored in the array should be String or Symbol.
+        #   Using this option, job_id is implicitly added to the keys.
+
         def initialize(options = {})
-          options[:config_file] ||= config_file if config_file.exist?
+          options[:config_file] ||= config_file if File.exist?(config_file)
           options = DEFAULTS
-             .merge(file_options(options))
-             .merge(options)
+                    .merge(file_options(options))
+                    .merge(options)
           set_attributes(options)
         end
 
+        def excluded_deduplication_keys=(keys)
+          @excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
+        end
+
         def client
-          @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
+          @client ||= begin
+            client = Aws::SQS::Client.new
+            client.config.user_agent_frameworks << 'aws-sdk-rails'
+            client
+          end
         end
 
         # Return the queue_url for a given job_queue name
@@ -111,9 +125,9 @@ module Aws
         # @api private
         def to_h
           h = {}
-          self.instance_variables.each do |v|
+          instance_variables.each do |v|
             v_sym = v.to_s.gsub('@', '').to_sym
-            val = self.instance_variable_get(v)
+            val = instance_variable_get(v)
             h[v_sym] = val
           end
           h
@@ -123,8 +137,9 @@ module Aws
 
         # Set accessible attributes after merged options.
         def set_attributes(options)
-          options.keys.each do |opt_name|
+          options.each_key do |opt_name|
             instance_variable_set("@#{opt_name}", options[opt_name])
+            client.config.user_agent_frameworks << 'aws-sdk-rails' if opt_name == :client
           end
         end
 
@@ -139,7 +154,7 @@ module Aws
 
         def config_file
           file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
-          file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
+          file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless File.exist?(file)
           file
         end
 
@@ -151,24 +166,22 @@ module Aws
 
         # @return [String] Configuration path found in environment or YAML file.
         def config_file_path(options)
-          options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
-        end
-
-        def user_agent
-          "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
+          options[:config_file] || ENV.fetch('AWS_SQS_ACTIVE_JOB_CONFIG_FILE', nil)
         end
 
         def load_yaml(file_path)
-          require "erb"
+          require 'erb'
           source = ERB.new(File.read(file_path)).result
 
           # Avoid incompatible changes with Psych 4.0.0
           # https://bugs.ruby-lang.org/issues/17866
+          # rubocop:disable Security/YAMLLoad
           begin
             YAML.load(source, aliases: true) || {}
           rescue ArgumentError
             YAML.load(source) || {}
           end
+          # rubocop:enable Security/YAMLLoad
         end
       end
     end

data/lib/aws/rails/sqs_active_job/deduplication.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    # SQS ActiveJob modules
+    module SqsActiveJob
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :excluded_deduplication_keys
+      end
+
+      # class methods for SQS ActiveJob.
+      module ClassMethods
+        def deduplicate_without(*keys)
+          self.excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/executor.rb

@@ -7,18 +7,17 @@ module Aws
     module SqsActiveJob
       # CLI runner for polling for SQS ActiveJobs
       class Executor
-
         DEFAULTS = {
-           min_threads:     0,
-           max_threads:     Concurrent.processor_count,
-           auto_terminate:  true,
-           idletime:        60, # 1 minute
-           fallback_policy: :caller_runs # slow down the producer thread
+          min_threads: 0,
+          max_threads: Concurrent.processor_count,
+          auto_terminate: true,
+          idletime: 60, # 1 minute
+          fallback_policy: :caller_runs # slow down the producer thread
         }.freeze
 
         def initialize(options = {})
           @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
-          @logger = options[:logger] || ActiveSupport::Logger.new(STDOUT)
+          @logger = options[:logger] || ActiveSupport::Logger.new($stdout)
         end
 
         # TODO: Consider catching the exception and sleeping instead of using :caller_runs
@@ -40,16 +39,16 @@ module Aws
           end
         end
 
-        def shutdown(timeout=nil)
+        def shutdown(timeout = nil)
           @executor.shutdown
           clean_shutdown = @executor.wait_for_termination(timeout)
           if clean_shutdown
             @logger.info 'Clean shutdown complete.  All executing jobs finished.'
           else
-            @logger.info "Timeout (#{timeout}) exceeded.  Some jobs may not have"\
-              " finished cleanly.  Unfinished jobs will not be removed from"\
-              " the queue and can be ru-run once their visibility timeout"\
-              " passes."
+            @logger.info "Timeout (#{timeout}) exceeded.  Some jobs may not have " \
+                         'finished cleanly.  Unfinished jobs will not be removed from ' \
+                         'the queue and can be ru-run once their visibility timeout ' \
+                         'passes.'
           end
         end
       end

data/lib/aws/rails/sqs_active_job/job_runner.rb

@@ -3,7 +3,6 @@
 module Aws
   module Rails
     module SqsActiveJob
-
       class JobRunner
         attr_reader :id, :class_name
 

data/lib/aws/rails/sqs_active_job/lambda_handler.rb

@@ -5,7 +5,6 @@ require 'aws-sdk-sqs'
 module Aws
   module Rails
     module SqsActiveJob
-
       # A lambda event handler to run jobs from an SQS queue trigger
       # Trigger the lambda from your SQS queue
       # Configure the entrypoint to: +config/environment.Aws::Rails::SqsActiveJob.lambda_job_handler+
@@ -23,13 +22,11 @@ module Aws
         "Processed #{event['Records'].length} jobs."
       end
 
-      private
-
       def self.to_sqs_msg(record)
         msg = Aws::SQS::Types::Message.new(
           body: record['body'],
           md5_of_body: record['md5OfBody'],
-          message_attributes: self.to_message_attributes(record),
+          message_attributes: to_message_attributes(record),
           message_id: record['messageId'],
           receipt_handle: record['receiptHandle']
         )

data/lib/aws/rails/sqs_active_job/poller.rb

@@ -7,19 +7,17 @@ require 'concurrent'
 module Aws
   module Rails
     module SqsActiveJob
-
-      class Interrupt < Exception; end
+      class Interrupt < StandardError; end
 
       # CLI runner for polling for SQS ActiveJobs
       # Use `aws_sqs_active_job --help` for detailed usage
       class Poller
-
         DEFAULT_OPTS = {
-          threads: 2*Concurrent.processor_count,
+          threads: 2 * Concurrent.processor_count,
           max_messages: 10,
           shutdown_timeout: 15,
           backpressure: 10
-        }
+        }.freeze
 
         def initialize(args = ARGV)
           @options = parse_args(args)
@@ -28,7 +26,7 @@ module Aws
         end
 
         def set_environment
-          @environment = @options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+          @environment = @options[:environment] || ENV['APP_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
         end
 
         def run
@@ -42,10 +40,9 @@ module Aws
                      .merge(@options.to_h)
           validate_config
           # ensure we have a logger configured
-          @logger = @options[:logger] || ActiveSupport::Logger.new(STDOUT)
+          @logger = @options[:logger] || ActiveSupport::Logger.new($stdout)
           @logger.info("Starting Poller with options=#{@options}")
 
-
           Signal.trap('INT') { raise Interrupt }
           Signal.trap('TERM') { raise Interrupt }
           @executor = Executor.new(max_threads: @options[:threads], logger: @logger, max_queue: @options[:backpressure])
@@ -78,9 +75,7 @@ module Aws
           # in order
           # Jobs with different message_group_id will be processed in
           # parallel and may be out of order.
-          if Aws::Rails::SqsActiveJob.fifo?(queue_url)
-            poller_options[:max_number_of_messages] = 1
-          end
+          poller_options[:max_number_of_messages] = 1 if Aws::Rails::SqsActiveJob.fifo?(queue_url)
 
           single_message = poller_options[:max_number_of_messages] == 1
 
@@ -89,35 +84,53 @@ module Aws
             @logger.info "Processing batch of #{msgs.length} messages"
             msgs.each do |msg|
               @executor.execute(Aws::SQS::Message.new(
-                queue_url: queue_url,
-                receipt_handle: msg.receipt_handle,
-                data: msg,
-                client: client
-              ))
+                                  queue_url: queue_url,
+                                  receipt_handle: msg.receipt_handle,
+                                  data: msg,
+                                  client: client
+                                ))
             end
           end
         end
 
         def boot_rails
           ENV['RACK_ENV'] = ENV['RAILS_ENV'] = @environment
-          require "rails"
-          require File.expand_path("config/environment.rb")
+          require 'rails'
+          require File.expand_path('config/environment.rb')
         end
 
         def parse_args(argv)
           out = {}
-          parser = ::OptionParser.new { |opts|
-            opts.on("-q", "--queue STRING", "[Required] Queue to poll") { |a| out[:queue] = a }
-            opts.on("-e", "--environment STRING", "Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.") { |a| out[:environment] = a }
-            opts.on("-t", "--threads INTEGER", Integer, "The maximum number of worker threads to create.  Defaults to 2x the number of processors available on this system.") { |a| out[:threads] = a }
-            opts.on("-b", "--backpressure INTEGER", Integer, "The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.") { |a| out[:backpressure] = a }
-            opts.on("-m", "--max_messages INTEGER", Integer, "Max number of messages to receive in a batch from SQS.") { |a| out[:max_messages] = a }
-            opts.on("-v", "--visibility_timeout INTEGER", Integer, "The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.") { |a| out[:visibility_timeout] = a }
-            opts.on("-s", "--shutdown_timeout INTEGER", Integer, "The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.") { |a| out[:shutdown_timeout] = a }
-          }
+          parser = ::OptionParser.new do |opts|
+            opts.on('-q', '--queue STRING', '[Required] Queue to poll') { |a| out[:queue] = a }
+            opts.on('-e', '--environment STRING',
+                    'Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.') do |a|
+              out[:environment] = a
+            end
+            opts.on('-t', '--threads INTEGER', Integer,
+                    'The maximum number of worker threads to create.  Defaults to 2x the number of processors available on this system.') do |a|
+              out[:threads] = a
+            end
+            opts.on('-b', '--backpressure INTEGER', Integer,
+                    'The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.') do |a|
+              out[:backpressure] = a
+            end
+            opts.on('-m', '--max_messages INTEGER', Integer,
+                    'Max number of messages to receive in a batch from SQS.') do |a|
+              out[:max_messages] = a
+            end
+            opts.on('-v', '--visibility_timeout INTEGER', Integer,
+                    'The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.') do |a|
+              out[:visibility_timeout] = a
+            end
+            opts.on('-s', '--shutdown_timeout INTEGER', Integer,
+                    'The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.') do |a|
+              out[:shutdown_timeout] = a
+            end
+          end
 
-          parser.banner = "aws_sqs_active_job [options]"
-          parser.on_tail "-h", "--help", "Show help" do
+          parser.banner = 'aws_sqs_active_job [options]'
+          parser.on_tail '-h', '--help', 'Show help' do
             puts parser
             exit 1
           end

data/lib/aws-sdk-rails.rb

@@ -5,6 +5,7 @@ require_relative 'aws/rails/sesv2_mailer'
 require_relative 'aws/rails/railtie'
 require_relative 'aws/rails/notifications'
 require_relative 'aws/rails/sqs_active_job/configuration'
+require_relative 'aws/rails/sqs_active_job/deduplication'
 require_relative 'aws/rails/sqs_active_job/executor'
 require_relative 'aws/rails/sqs_active_job/job_runner'
 require_relative 'aws/rails/sqs_active_job/lambda_handler'