aws-activejob-sqs 0.1.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1eeedcef79ed4252667f821bcd84cef11304edc53368206191e415a75c6d9f6e
-  data.tar.gz: 7478443ea1a47d862e217a9f5ce44b81cd8c96bfc9a6360b5c48b6681c71f39f
+  metadata.gz: d11b67b0f50d2a026a3c64f8289d98c8b8a6cd37e5a670346807a29061f183a0
+  data.tar.gz: 503d0cbb51a915877b89dcbcf23a97a62fbf50599daddb56120e010dc4e8b612
 SHA512:
-  metadata.gz: f61a2e9c1d5065feadde7833499909449cc1ce390569e5528fca0633315364b24dc27d87cee3a9b39d98b6081a7405e2537584a8713119df68db6b3b87d03e1f
-  data.tar.gz: a75662933ed22c33465d209fe95034a1c874b38fdcc50e595417d04f025b870a0e7054fdf007bd53e19c461519a193e6996c830e1d1654d7a34a8224f75a9530
+  metadata.gz: 69a4e6b7028efff40930d0f2d2ee76131878399b92d9ea1543a7461c624ef7b2d83d43a8243877a41321af428e3e65ec9e448cfeea0f378903609704110f9f59
+  data.tar.gz: 3375ea92eb2f0923eca14e90a326210c027bde82879933fa6fd4bbd54270aeb1519188fcf85a11d28b41731b7090f79d82796b543dfe6b208ecbb8dd4956d492

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+1.0.0 (2024-12-13)
+------------------
+
+* Feature - Support polling on multiple queues. (#4)
+* Feature - Support running without Rails. (#5)
+* Feature - Replace `retry_standard_errors` with `poller_error_handler`. (#6)
+* Feature - Support per queue configuration. (#4)
+* Feature - Support loading global and queue specific configuration from ENV. (#3)
+
 0.1.1 (2024-12-02)
 ------------------
 

data/VERSION

@@ -1 +1 @@
-0.1.1
+1.0.0

data/bin/aws_active_job_sqs

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/aws-activejob-sqs'
+require_relative '../lib/aws/active_job/sqs/cli_options'
+require_relative '../lib/aws/active_job/sqs/poller'
+
+opts = Aws::ActiveJob::SQS::CliOptions.parse(ARGV)
+
+if opts[:boot_rails]
+  require 'rails'
+  require File.expand_path('config/environment.rb')
+end
+
+require File.join(Dir.pwd, opts[:require]) if opts[:require]
+
+Aws::ActiveJob::SQS::Poller.new(opts.to_h.compact).run

data/lib/active_job/queue_adapters/sqs_adapter/params.rb

@@ -22,7 +22,7 @@ module ActiveJob
         end
 
         def queue_url
-          @queue_url ||= Aws::ActiveJob::SQS.config.queue_url_for(@job.queue_name)
+          @queue_url ||= Aws::ActiveJob::SQS.config.url_for(@job.queue_name)
         end
 
         def entry
@@ -37,7 +37,7 @@ module ActiveJob
 
         def default_entry
           {
-            message_body: Aws::Json.dump(@body),
+            message_body: ActiveSupport::JSON.dump(@body),
             message_attributes: message_attributes
           }
         end
@@ -58,10 +58,10 @@ module ActiveJob
         def options_for_fifo
           options = {}
           options[:message_deduplication_id] =
-            Digest::SHA256.hexdigest(Aws::Json.dump(deduplication_body))
+            Digest::SHA256.hexdigest(ActiveSupport::JSON.dump(deduplication_body))
 
           message_group_id = @job.message_group_id if @job.respond_to?(:message_group_id)
-          message_group_id ||= Aws::ActiveJob::SQS.config.message_group_id
+          message_group_id ||= Aws::ActiveJob::SQS.config.message_group_id_for(@job.queue_name)
 
           options[:message_group_id] = message_group_id
           options
@@ -69,7 +69,7 @@ module ActiveJob
 
         def deduplication_body
           ex_dedup_keys = @job.excluded_deduplication_keys if @job.respond_to?(:excluded_deduplication_keys)
-          ex_dedup_keys ||= Aws::ActiveJob::SQS.config.excluded_deduplication_keys
+          ex_dedup_keys ||= Aws::ActiveJob::SQS.config.excluded_deduplication_keys_for(@job.queue_name)
 
           @body.except(*ex_dedup_keys)
         end

data/lib/active_job/queue_adapters/sqs_adapter.rb

@@ -8,7 +8,7 @@ module ActiveJob
     #
     # To use this adapter, set up as:
     #
-    #     config.active_job.queue_adapter = :sqs_async
+    #     config.active_job.queue_adapter = :sqs
     class SqsAdapter
       def enqueue_after_transaction_commit?
         # can be removed after Rails 8
@@ -27,29 +27,40 @@ module ActiveJob
       def enqueue_all(jobs)
         enqueued_count = 0
         jobs.group_by(&:queue_name).each do |queue_name, same_queue_jobs|
-          queue_url = Aws::ActiveJob::SQS.config.queue_url_for(queue_name)
-          base_send_message_opts = { queue_url: queue_url }
-
-          same_queue_jobs.each_slice(10) do |chunk|
-            entries = chunk.map do |job|
-              entry = Params.new(job, nil).entry
-              entry[:id] = job.job_id
-              entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
-              entry
-            end
-
-            send_message_opts = base_send_message_opts.deep_dup
-            send_message_opts[:entries] = entries
-
-            send_message_batch_result = Aws::ActiveJob::SQS.config.client.send_message_batch(send_message_opts)
-            enqueued_count += send_message_batch_result.successful.count
-          end
+          enqueued_count += enqueue_batches(queue_name, same_queue_jobs)
         end
         enqueued_count
       end
 
       private
 
+      def enqueue_batches(queue_name, same_queue_jobs)
+        enqueued_count = 0
+        queue_url = Aws::ActiveJob::SQS.config.url_for(queue_name)
+
+        same_queue_jobs.each_slice(10) do |chunk|
+          enqueued_count += enqueue_batch(queue_url, chunk)
+        end
+        enqueued_count
+      end
+
+      def enqueue_batch(queue_url, chunk)
+        entries = chunk.map do |job|
+          entry = Params.new(job, nil).entry
+          entry[:id] = job.job_id
+          entry[:delay_seconds] = Params.assured_delay_seconds(job.scheduled_at) if job.scheduled_at
+          entry
+        end
+
+        send_message_opts = {
+          queue_url: queue_url,
+          entries: entries
+        }
+
+        send_message_batch_result = Aws::ActiveJob::SQS.config.client.send_message_batch(send_message_opts)
+        send_message_batch_result.successful.count
+      end
+
       def _enqueue(job, body = nil, send_message_opts = {})
         body ||= job.serialize
         params = Params.new(job, body)

data/lib/active_job/queue_adapters/sqs_async_adapter.rb

@@ -19,7 +19,7 @@ module ActiveJob
 
       def _enqueue(job, body = nil, send_message_opts = {})
         # FIFO jobs must be queued in order, so do not queue async
-        queue_url = Aws::ActiveJob::SQS.config.queue_url_for(job.queue_name)
+        queue_url = Aws::ActiveJob::SQS.config.url_for(job.queue_name)
         if Aws::ActiveJob::SQS.fifo?(queue_url)
           super
         else
@@ -29,8 +29,7 @@ module ActiveJob
           Concurrent::Promises
             .future { super(job, body, send_message_opts) }
             .rescue do |e|
-              # TODO: should be config logger? fails
-              Rails.logger.error "Failed to queue job #{job}. Reason: #{e}"
+              Aws::ActiveJob::SQS.config.logger.error "Failed to queue job #{job}. Reason: #{e}"
               error_handler = Aws::ActiveJob::SQS.config.async_queue_error_handler
               error_handler&.call(e, job, send_message_opts)
             end

data/lib/aws/active_job/sqs/cli_options.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module Aws
+  module ActiveJob
+    module SQS
+      # options for the aws_active_job_sqs CLI.
+      # @api private
+      module CliOptions
+        def self.option_parser(out)
+          ::OptionParser.new do |opts|
+            queues_option(opts, out)
+            threads_option(opts, out)
+            backpressure_option(opts, out)
+            max_messages_option(opts, out)
+            visibility_timeout_option(opts, out)
+            shutdown_timeout_option(opts, out)
+            boot_rails_option(opts, out)
+            require_option(opts, out)
+          end
+        end
+        private_class_method :option_parser
+
+        def self.parse(argv)
+          out = { boot_rails: true, queues: [] }
+          parser = option_parser(out)
+
+          parser.banner = 'aws_active_job_sqs [options]'
+          parser.on_tail '-h', '--help', 'Show help' do
+            puts parser
+            exit 1
+          end
+
+          parser.parse(argv)
+          out
+        end
+
+        def self.require_option(opts, out)
+          doc = 'Additional file to require before starting the poller.  ' \
+                'Can be used to define/load job classes with --no-rails.'
+          opts.on('-r', '--require STRING', String, doc) do |a|
+            out[:require] = a
+          end
+        end
+
+        def self.boot_rails_option(opts, out)
+          doc = 'When set boots rails before running the poller.'
+          opts.on('--[no-]rails [FLAG]', TrueClass, doc) do |a|
+            out[:boot_rails] = a.nil? ? true : a
+          end
+        end
+
+        def self.shutdown_timeout_option(opts, out)
+          doc = 'The amount of time to wait for a clean shutdown.  Jobs that ' \
+                'are unable to complete in this time will not be deleted from ' \
+                'the SQS queue and will be retryable after the visibility timeout.'
+          opts.on('-s', '--shutdown_timeout INTEGER', Integer, doc) do |a|
+            out[:shutdown_timeout] = a
+          end
+        end
+
+        def self.visibility_timeout_option(opts, out)
+          doc = 'The visibility timeout is the number of seconds that a ' \
+                'message will not be processable by any other consumers. ' \
+                'You should set this value to be longer than your expected ' \
+                'job runtime to prevent other processes from picking up an ' \
+                'running job.  See the SQS Visibility Timeout Documentation ' \
+                'at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/' \
+                'SQSDeveloperGuide/sqs-visibility-timeout.html.'
+          opts.on('-v', '--visibility_timeout INTEGER', Integer, doc) do |a|
+            out[:visibility_timeout] = a
+          end
+        end
+
+        def self.max_messages_option(opts, out)
+          doc = 'Max number of messages to receive in a batch from SQS.'
+          opts.on('-m', '--max_messages INTEGER', Integer, doc) do |a|
+            out[:max_messages] = a
+          end
+        end
+
+        def self.backpressure_option(opts, out)
+          doc = 'The maximum number of messages to have waiting in ' \
+                'the Executor queue. This should be a low, but non zero number.  ' \
+                'Messages in the Executor queue cannot be picked up by other ' \
+                'processes and will slow down shutdown.'
+          opts.on('-b', '--backpressure INTEGER', Integer, doc) do |a|
+            out[:backpressure] = a
+          end
+        end
+
+        def self.threads_option(opts, out)
+          doc = 'The maximum number of worker threads to create.  ' \
+                'Defaults to 2x the number of processors available on this system.'
+          opts.on('-t', '--threads INTEGER', Integer, doc) do |a|
+            out[:threads] = a
+          end
+        end
+
+        def self.queues_option(opts, out)
+          doc = 'Queue(s) to poll. You may specify this argument multiple ' \
+                'times to poll multiple queues.  If not specified, will ' \
+                'start pollers for all queues defined.'
+          opts.on('-q', '--queue STRING', doc) do |a|
+            out[:queues] << a.to_sym
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws/active_job/sqs/configuration.rb

@@ -3,34 +3,98 @@
 module Aws
   module ActiveJob
     module SQS
-      # Use +Aws::ActiveJob::SQS.config+ to access the singleton config instance.
+      # This class provides a Configuration object for AWS ActiveJob
+      # by pulling configuration options from runtime code, the ENV, a YAML file,
+      # and default settings, in that order. Values set on queues are used
+      # preferentially to global values.
+      #
+      # Use {Aws::ActiveJob::SQS.config Aws::ActiveJob::SQS.config}
+      # to access the singleton config instance and use
+      # {Aws::ActiveJob::SQS.configure Aws::ActiveJob::SQS.configure} to
+      # configure in code:
+      #
+      #     Aws::ActiveJob::SQS.configure do |config|
+      #       config.logger = Rails.logger
+      #       config.max_messages = 5
+      #     end
+      #
+      # # Configuation YAML File
+      # By default, this class will load configuration from the
+      # `config/aws_active_job_sqs/<RAILS_ENV}.yml` or
+      # `config/aws_active_job_sqs.yml` files.  You may specify the file used
+      # through the `:config_file` option in code or the
+      # `AWS_ACTIVE_JOB_SQS_CONFIG_FILE` environment variable.
+      #
+      # # Global and queue specific options
+      # Values configured for specific queues are used preferentially to
+      # global values. See: {QUEUE_CONFIGS} for supported queue specific
+      # options.
+      #
+      # # Environment Variables
+      # The Configuration loads global and qubeue specific values from your
+      # environment. Global keys take the form of:
+      # `AWS_ACTIVE_JOB_SQS_<KEY_NAME>` and queue specific keys take the
+      # form of: `AWS_ACTIVE_JOB_SQS_<QUEUE_NAME>_<KEY_NAME>`.
+      # <QUEUE_NAME> is case-insensitive and is always down cased. Configuring
+      # non-snake case queues (containing upper case) through ENV is
+      # not supported.
+      #
+      # Example:
+      #
+      #     export AWS_ACTIVE_JOB_SQS_MAX_MESSAGES = 5
+      #     export AWS_ACTIVE_JOB_SQS_DEFAULT_URL = https://my-queue.aws
+      #
+      # For supported global ENV configurations see
+      # {GLOBAL_ENV_CONFIGS}.  For supported queue specific ENV configurations
+      # see: {QUEUE_ENV_CONFIGS}.
+      #
       class Configuration
         # Default configuration options
         # @api private
         DEFAULTS = {
+          threads: 2 * Concurrent.processor_count,
+          backpressure: 10,
           max_messages: 10,
           shutdown_timeout: 15,
-          retry_standard_errors: true, # TODO: Remove in next MV
-          queues: {},
-          logger: ::Rails.logger,
-          message_group_id: 'SqsActiveJobGroup',
+          queues: Hash.new { |h, k| h[k] = {} },
+          message_group_id: 'ActiveJobSqsGroup',
           excluded_deduplication_keys: ['job_id']
         }.freeze
 
-        # @api private
-        attr_accessor :queues, :max_messages, :visibility_timeout,
-                      :shutdown_timeout, :client, :logger,
-                      :async_queue_error_handler, :message_group_id
+        GLOBAL_ENV_CONFIGS = %i[
+          config_file
+          threads
+          backpressure
+          max_messages
+          shutdown_timeout
+          visibility_timeout
+          message_group_id
+        ].freeze
+
+        QUEUE_ENV_CONFIGS = %i[
+          url
+          max_messages
+          visibility_timeout
+          message_group_id
+        ].freeze
+
+        QUEUE_CONFIGS = QUEUE_ENV_CONFIGS + %i[excluded_deduplication_keys]
 
-        attr_reader :excluded_deduplication_keys
+        QUEUE_KEY_REGEX =
+          /AWS_ACTIVE_JOB_SQS_([\w]+)_(#{QUEUE_ENV_CONFIGS.map(&:upcase).join('|')})/.freeze
 
-        # Don't use this method directly: Configuration is a singleton class, use
-        # +Aws::ActiveJob::SQS.config+ to access the singleton config.
+        # Don't use this method directly: Configuration is a singleton class,
+        # use {Aws::ActiveJob::SQS.config Aws::ActiveJob::SQS.config}
+        # to access the singleton config instance and use
+        # {Aws::ActiveJob::SQS.configure Aws::ActiveJob::SQS.configure} to
+        # configure in code:
         #
         # @param [Hash] options
-        # @option options [Hash[Symbol, String]] :queues A mapping between the
-        #   active job queue name and the SQS Queue URL. Note: multiple active
-        #   job queues can map to the same SQS Queue URL.
+        # @option options [Hash<Symbol, Hash>] :queues A mapping between the
+        #   active job queue name and the queue properties. Values
+        #   configured on the queue are used preferentially to the global
+        #   values. See: {QUEUE_CONFIGS} for supported queue specific options.
+        #   Note: multiple active job queues can map to the same SQS Queue URL.
         #
         # @option options  [Integer] :max_messages
         #    The max number of messages to poll for in a batch.
@@ -50,22 +114,22 @@ module Aws
         #   will not be deleted from the SQS queue and will be retryable after
         #   the visibility timeout.
         #
-        # @ option options [Boolean] :retry_standard_errors
-        #   If `true`, StandardErrors raised by ActiveJobs are left on the queue
-        #   and will be retried (pending the SQS Queue's redrive/DLQ/maximum receive settings).
-        #   This behavior overrides the standard Rails ActiveJob
-        #   [Retry/Discard for failed jobs](https://guides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs)
-        #   behavior.  When set to `true` the retries provided by this will be
-        #   on top of any retries configured on the job with `retry_on`.
-        #   When `false`, retry behavior is fully configured
-        #   through `retry_on`/`discard_on` on the ActiveJobs.
+        # @option options [Callable] :poller_error_handler and error handler to
+        #   be called when the poller encounters an error running a job.  Called
+        #   with exception, sqs_message. You may re-raise the exception to
+        #   terminate the poller. You may also choose whether to delete the
+        #   sqs_message or not.  If the message is not explicitly deleted
+        #   then the message will be left on the queue and will be
+        #   retried (pending the SQS Queue's redrive/DLQ/maximum
+        #   receive settings). Retries provided by this mechanism are
+        #   after any retries configured on the job with `retry_on`.
         #
         # @option options [ActiveSupport::Logger] :logger Logger to use
         #   for the poller.
         #
         # @option options [String] :config_file
         #   Override file to load configuration from. If not specified will
-        #   attempt to load from config/aws_sqs_active_job.yml.
+        #   attempt to load from config/aws_active_job_sqs.yml.
         #
         # @option options [String] :message_group_id (SqsActiveJobGroup)
         #  The message_group_id to use for queueing messages on a fifo queues.
@@ -73,7 +137,7 @@ module Aws
         #  See the (SQS FIFO Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html]
         #
         # @option options [Callable] :async_queue_error_handler An error handler
-        #   to be called when the async active job adapter experiances an error
+        #   to be called when the async active job adapter experiences an error
         #   queueing a job.  Only applies when
         #   +active_job.queue_adapter = :sqs_async+.  Called with:
         #   [error, job, job_options]
@@ -86,17 +150,31 @@ module Aws
         #   Using this option, job_id is implicitly added to the keys.
 
         def initialize(options = {})
-          options[:config_file] ||= config_file if File.exist?(config_file)
-          options = DEFAULTS
-                    .merge(file_options(options))
-                    .merge(options)
-          set_attributes(options)
+          opts = env_options.deep_merge(options)
+          opts = file_options(opts).deep_merge(opts)
+          opts = DEFAULTS.merge(logger: default_logger).deep_merge(opts)
+
+          apply_attributes(opts)
         end
 
+        # @api private
+        attr_accessor :queues, :threads, :backpressure,
+                      :shutdown_timeout, :logger,
+                      :async_queue_error_handler
+
+        # @api private
+        attr_writer :max_messages, :message_group_id, :visibility_timeout,
+                    :poller_error_handler, :client
+
         def excluded_deduplication_keys=(keys)
           @excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
         end
 
+        def poller_error_handler(&block)
+          @poller_error_handler = block if block_given?
+          @poller_error_handler
+        end
+
         def client
           @client ||= begin
             client = Aws::SQS::Client.new
@@ -105,12 +183,10 @@ module Aws
           end
         end
 
-        # Return the queue_url for a given job_queue name
-        def queue_url_for(job_queue)
-          job_queue = job_queue.to_sym
-          raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
-
-          queues[job_queue]
+        QUEUE_CONFIGS.each do |key|
+          define_method(:"#{key}_for") do |job_queue|
+            queue_attribute_for(key, job_queue)
+          end
         end
 
         # @api private
@@ -131,26 +207,68 @@ module Aws
 
         private
 
+        def queue_attribute_for(attribute, job_queue)
+          job_queue = job_queue.to_sym
+          raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
+
+          queues[job_queue][attribute] || instance_variable_get("@#{attribute}")
+        end
+
         # Set accessible attributes after merged options.
-        def set_attributes(options)
+        def apply_attributes(options)
           options.each_key do |opt_name|
             instance_variable_set("@#{opt_name}", options[opt_name])
             client.config.user_agent_frameworks << 'aws-activejob-sqs' if opt_name == :client
           end
         end
 
+        # resolve ENV for global and queue specific options
+        def env_options
+          resolved = { queues: {} }
+          resolve_global_env_options(resolved)
+          resolve_queue_env__options(resolved)
+          resolved
+        end
+
+        def resolve_queue_env__options(resolved)
+          ENV.each_key do |key|
+            next unless (match = QUEUE_KEY_REGEX.match(key))
+
+            queue_name = match[1].downcase.to_sym
+            resolved[:queues][queue_name] ||= {}
+            resolved[:queues][queue_name][match[2].downcase.to_sym] =
+              parse_env_value(key)
+          end
+        end
+
+        def resolve_global_env_options(resolved)
+          GLOBAL_ENV_CONFIGS.each do |cfg|
+            env_name = "AWS_ACTIVE_JOB_SQS_#{cfg.to_s.upcase}"
+            resolved[cfg] = parse_env_value(env_name) if ENV.key? env_name
+          end
+        end
+
+        def parse_env_value(key)
+          val = ENV.fetch(key, nil)
+          Integer(val)
+        rescue ArgumentError, TypeError
+          %w[true false].include?(val) ? val == 'true' : val
+        end
+
         def file_options(options = {})
-          file_path = config_file_path(options)
+          file_path = options[:config_file] || default_config_file
           if file_path
             load_from_file(file_path)
           else
-            {}
+            options
           end
         end
 
-        def config_file
-          file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
-          file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless File.exist?(file)
+        def default_config_file
+          return unless defined?(::Rails)
+
+          file = ::Rails.root.join("config/aws_active_job_sqs/#{::Rails.env}.yml")
+          file = ::Rails.root.join('config/aws_active_job_sqs.yml') unless File.exist?(file)
           file
         end
 
@@ -160,12 +278,9 @@ module Aws
           opts.deep_symbolize_keys
         end
 
-        # @return [String] Configuration path found in environment or YAML file.
-        def config_file_path(options)
-          options[:config_file] || ENV.fetch('AWS_SQS_ACTIVE_JOB_CONFIG_FILE', nil)
-        end
-
         def load_yaml(file_path)
+          return {} unless File.exist?(file_path)
+
           require 'erb'
           source = ERB.new(File.read(file_path)).result
 
@@ -177,6 +292,14 @@ module Aws
             YAML.safe_load(source) || {}
           end
         end
+
+        def default_logger
+          if defined?(::Rails)
+            ::Rails.logger
+          else
+            ActiveSupport::Logger.new($stdout)
+          end
+        end
       end
     end
   end

data/lib/aws/active_job/sqs/deduplication.rb

@@ -3,16 +3,19 @@
 module Aws
   module ActiveJob
     module SQS
-      extend ActiveSupport::Concern
+      # Mixin module to configure job level deduplication keys
+      module Deduplication
+        extend ActiveSupport::Concern
 
-      included do
-        class_attribute :excluded_deduplication_keys
-      end
+        included do
+          class_attribute :excluded_deduplication_keys
+        end
 
-      # class methods for SQS ActiveJob.
-      module ClassMethods
-        def deduplicate_without(*keys)
-          self.excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
+        # class methods for SQS ActiveJob.
+        module ClassMethods
+          def deduplicate_without(*keys)
+            self.excluded_deduplication_keys = keys.map(&:to_s) | ['job_id']
+          end
         end
       end
     end

data/lib/aws/active_job/sqs/executor.rb

@@ -31,21 +31,27 @@ module Aws
 
         def initialize(options = {})
           @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
-          @retry_standard_errors = options[:retry_standard_errors]
           @logger = options[:logger] || ActiveSupport::Logger.new($stdout)
           @task_complete = Concurrent::Event.new
+          @post_mutex = Mutex.new
+
+          @error_handler = options[:error_handler]
+          @error_queue = Thread::Queue.new
+          @error_handler_thread = Thread.new(&method(:handle_errors))
+          @error_handler_thread.abort_on_exception = true
+          @error_handler_thread.report_on_exception = false
+          @shutting_down = Concurrent::AtomicBoolean.new(false)
         end
 
         def execute(message)
-          post_task(message)
-        rescue Concurrent::RejectedExecutionError
-          # no capacity, wait for a task to complete
-          @task_complete.reset
-          @task_complete.wait
-          retry
+          @post_mutex.synchronize do
+            _execute(message)
+          end
         end
 
         def shutdown(timeout = nil)
+          @shutting_down.make_true
+
           run_hooks_for(:stop)
           @executor.shutdown
           clean_shutdown = @executor.wait_for_termination(timeout)
@@ -57,41 +63,71 @@ module Aws
                          'the queue and can be ru-run once their visibility timeout ' \
                          'passes.'
           end
+          @error_queue.push(nil) # process any remaining errors and then terminate
+          @error_handler_thread.join unless @error_handler_thread == Thread.current
+          @shutting_down.make_false
         end
 
         private
 
+        def _execute(message)
+          post_task(message)
+        rescue Concurrent::RejectedExecutionError
+          # no capacity, wait for a task to complete
+          @task_complete.reset
+          @task_complete.wait
+          retry
+        end
+
         def post_task(message)
-          @executor.post(message) do |message|
-            job = JobRunner.new(message)
-            @logger.info("Running job: #{job.id}[#{job.class_name}]")
-            job.run
-            message.delete
-          rescue Aws::Json::ParseError => e
-            @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
-          rescue StandardError => e
-            job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
-            @logger.info "Error processing job #{job_msg}: #{e}"
-            @logger.debug e.backtrace.join("\n")
-
-            if @retry_standard_errors && !job.exception_executions?
-              @logger.info(
-                'retry_standard_errors is enabled and job has not ' \
-                "been retried by Rails.  Leaving #{job_msg} in the queue."
-              )
-            else
-              message.delete
-            end
-          ensure
-            @task_complete.set
+          @executor.post(message) do |msg|
+            execute_task(msg)
           end
         end
 
+        def execute_task(message)
+          job = JobRunner.new(message)
+          @logger.info("Running job: #{job.id}[#{job.class_name}]")
+          job.run
+          message.delete
+        rescue JSON::ParserError => e
+          @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
+        rescue StandardError => e
+          handle_standard_error(e, job, message)
+        ensure
+          @task_complete.set
+        end
+
+        def handle_standard_error(error, job, message)
+          job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
+          @logger.info "Error processing job #{job_msg}: #{error}"
+          @logger.debug error.backtrace.join("\n")
+
+          @error_queue.push([error, message])
+        end
+
         def run_hooks_for(event_name)
           return unless (hooks = self.class.lifecycle_hooks[event_name])
 
           hooks.each(&:call)
         end
+
+        # run in the @error_handler_thread
+        def handle_errors
+          # wait until errors are placed in the error queue
+          while ((exception, message) = @error_queue.pop)
+            raise exception unless @error_handler
+
+            @error_handler.call(exception, message)
+
+          end
+        rescue StandardError => e
+          @logger.info("Unhandled exception executing jobs in poller: #{e}.")
+          @logger.info('Shutting down executor')
+          shutdown unless @shutting_down.true?
+
+          raise e # re-raise the error, terminating the application
+        end
       end
     end
   end

data/lib/aws/active_job/sqs/job_runner.rb

@@ -8,7 +8,7 @@ module Aws
         attr_reader :id, :class_name
 
         def initialize(message)
-          @job_data = Aws::Json.load(message.data.body)
+          @job_data = ActiveSupport::JSON.load(message.data.body)
           @class_name = @job_data['job_class'].constantize
           @id = @job_data['job_id']
         end

data/lib/aws/active_job/sqs/lambda_handler.rb

@@ -5,60 +5,65 @@ require 'aws-sdk-sqs'
 module Aws
   module ActiveJob
     module SQS
-      class << self
-        # A lambda event handler to run jobs from an SQS queue trigger.
-        # Configure the entrypoint to: +config/environment.Aws::ActiveJob::SQS.lambda_job_handler+
-        # This will load your Rails environment, and then use this method as the handler.
-        def lambda_job_handler(event:, context:)
-          return 'no records to process' unless event['Records']
+      # Lambda event handler to run jobs from an SQS queue trigger
+      module LambdaHandler
+        class << self
+          # A lambda event handler to run jobs from an SQS queue trigger.
+          # Configure the entrypoint to: +config/environment.Aws::ActiveJob::SQS::LambdaHandler.job_handler+
+          # This will load your Rails environment, and then use this method as the handler.
+          def job_handler(event:, context:)
+            return 'no records to process' unless event['Records']
 
-          event['Records'].each do |record|
-            sqs_msg = to_sqs_msg(record)
-            job = Aws::ActiveJob::SQS::JobRunner.new(sqs_msg)
-            puts("Running job: #{job.id}[#{job.class_name}]")
-            job.run
-            sqs_msg.delete
+            puts "job_handler running for #{event} with context: #{context}"
+
+            event['Records'].each do |record|
+              sqs_msg = to_sqs_msg(record)
+              job = Aws::ActiveJob::SQS::JobRunner.new(sqs_msg)
+              puts "Running job: #{job.id}[#{job.class_name}]"
+              job.run
+              sqs_msg.delete
+            end
+            "Processed #{event['Records'].length} jobs."
           end
-          "Processed #{event['Records'].length} jobs."
-        end
 
-        private
+          private
 
-        def to_sqs_msg(record)
-          msg = Aws::SQS::Types::Message.new(
-            body: record['body'],
-            md5_of_body: record['md5OfBody'],
-            message_attributes: to_message_attributes(record),
-            message_id: record['messageId'],
-            receipt_handle: record['receiptHandle']
-          )
-          Aws::SQS::Message.new(
-            queue_url: to_queue_url(record),
-            receipt_handle: msg.receipt_handle,
-            data: msg,
-            client: Aws::ActiveJob::SQS.config.client
-          )
-        end
+          def to_sqs_msg(record)
+            msg = Aws::SQS::Types::Message.new(
+              body: record['body'],
+              md5_of_body: record['md5OfBody'],
+              message_attributes: to_message_attributes(record),
+              message_id: record['messageId'],
+              receipt_handle: record['receiptHandle']
+            )
+            Aws::SQS::Message.new(
+              queue_url: to_queue_url(record),
+              receipt_handle: msg.receipt_handle,
+              data: msg,
+              client: Aws::ActiveJob::SQS.config.client
+            )
+          end
 
-        def to_message_attributes(record)
-          record['messageAttributes'].transform_values do |value|
-            {
-              string_value: value['stringValue'],
-              binary_value: value['binaryValue'],
-              string_list_values: ['stringListValues'],
-              binary_list_values: value['binaryListValues'],
-              data_type: value['dataType']
-            }
+          def to_message_attributes(record)
+            record['messageAttributes'].transform_values do |value|
+              {
+                string_value: value['stringValue'],
+                binary_value: value['binaryValue'],
+                string_list_values: ['stringListValues'],
+                binary_list_values: value['binaryListValues'],
+                data_type: value['dataType']
+              }
+            end
           end
-        end
 
-        def to_queue_url(record)
-          source_arn = record['eventSourceARN']
-          raise ArgumentError, "Invalid queue arn: #{source_arn}" unless Aws::ARNParser.arn?(source_arn)
+          def to_queue_url(record)
+            source_arn = record['eventSourceARN']
+            raise ArgumentError, "Invalid queue arn: #{source_arn}" unless Aws::ARNParser.arn?(source_arn)
 
-          arn = Aws::ARNParser.parse(source_arn)
-          sfx = Aws::Partitions::EndpointProvider.dns_suffix_for(arn.region)
-          "https://sqs.#{arn.region}.#{sfx}/#{arn.account_id}/#{arn.resource}"
+            arn = Aws::ARNParser.parse(source_arn)
+            sfx = Aws::Partitions::EndpointProvider.dns_suffix_for(arn.region)
+            "https://sqs.#{arn.region}.#{sfx}/#{arn.account_id}/#{arn.resource}"
+          end
         end
       end
     end

data/lib/aws/active_job/sqs/poller.rb

@@ -1,158 +1,151 @@
 # frozen_string_literal: true
 
 require 'aws-sdk-sqs'
-require 'optparse'
 require 'concurrent'
 
 module Aws
   module ActiveJob
     module SQS
       # CLI runner for polling for SQS ActiveJobs
-      # Use `aws_sqs_active_job --help` for detailed usage
+      # Use `aws_active_job_sqs --help` for detailed usage
       class Poller
         class Interrupt < StandardError; end
 
-        DEFAULT_OPTS = {
-          threads: 2 * Concurrent.processor_count,
-          max_messages: 10,
-          shutdown_timeout: 15,
-          backpressure: 10,
-          retry_standard_errors: true
-        }.freeze
-
-        def initialize(args = ARGV)
-          @options = parse_args(args)
-          # Set_environment must be run before we boot_rails
-          set_environment
-        end
-
-        def set_environment
-          @environment = @options[:environment] || ENV['APP_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+        def initialize(options = {})
+          @queues = options.delete(:queues)
+          @options = options
         end
 
         def run
-          # exit 0
-          boot_rails
-
-          # cannot load config (from file or initializers) until after
-          # rails has been booted.
-          @options = DEFAULT_OPTS
-                     .merge(Aws::ActiveJob::SQS.config.to_h)
-                     .merge(@options.to_h)
-          validate_config
-          # ensure we have a logger configured
-          @logger = @options[:logger] || ActiveSupport::Logger.new($stdout)
-          @logger.info("Starting Poller with options=#{@options}")
+          init_config
+
+          config = Aws::ActiveJob::SQS.config
 
           Signal.trap('INT') { raise Interrupt }
           Signal.trap('TERM') { raise Interrupt }
           @executor = Executor.new(
-            max_threads: @options[:threads],
+            max_threads: config.threads,
             logger: @logger,
-            max_queue: @options[:backpressure],
-            retry_standard_errors: @options[:retry_standard_errors]
+            max_queue: config.backpressure,
+            error_handler: config.poller_error_handler
           )
 
           poll
         rescue Interrupt
           @logger.info 'Process Interrupted or killed - attempting to shutdown cleanly.'
-          shutdown
+          shutdown(config.shutdown_timeout)
           exit
         end
 
         private
 
-        def shutdown
-          @executor.shutdown(@options[:shutdown_timeout])
+        def init_config
+          Aws::ActiveJob::SQS.configure do |cfg|
+            @options.each_pair do |key, value|
+              cfg.send(:"#{key}=", value) if cfg.respond_to?(:"#{key}=")
+            end
+          end
+
+          # ensure we have a logger configured
+          config = Aws::ActiveJob::SQS.config
+          @logger = config.logger || ActiveSupport::Logger.new($stdout)
+          @logger.info("Starting Poller with config=#{config.to_h}")
+        end
+
+        def shutdown(timeout)
+          @executor.shutdown(timeout)
         end
 
         def poll
-          queue_url = Aws::ActiveJob::SQS.config.queue_url_for(@options[:queue])
-          @logger.info "Polling on: #{@options[:queue]} => #{queue_url}"
-          client = Aws::ActiveJob::SQS.config.client
-          @poller = Aws::SQS::QueuePoller.new(queue_url, client: client)
+          config = Aws::ActiveJob::SQS.config
+          if @queues && !@queues.empty?
+            if @queues.size == 1
+              # single queue, use main thread
+              poll_foreground(@queues.first)
+            else
+              poll_background(@queues)
+            end
+          else
+            # poll on all configured queues
+            @logger.info("No queues specified - polling on all configured queues: #{config.queues.keys}")
+            poll_background(config.queues.keys)
+          end
+        end
+
+        def poll_foreground(queue)
+          config = Aws::ActiveJob::SQS.config
+          validate_config(queue)
+          queue_url = config.url_for(queue)
+
+          poller_options = poller_options(queue)
+          @logger.info "Foreground Polling on: #{queue} => #{queue_url} with options=#{poller_options}"
+
+          _poll(poller_options, queue_url)
+        end
+
+        def poll_background(queues)
+          config = Aws::ActiveJob::SQS.config
+          queues.each { |q| validate_config(q) }
+          poller_threads = queues.map do |queue|
+            Thread.new do
+              queue_url = config.url_for(queue)
+
+              poller_options = poller_options(queue)
+              @logger.info "Background Polling on: #{queue} => #{queue_url} with options=#{poller_options}"
+
+              _poll(poller_options, queue_url)
+            end
+          end
+          poller_threads.each(&:join)
+        end
+
+        def validate_config(queue)
+          return if Aws::ActiveJob::SQS.config.queues[queue]&.fetch(:url, nil)
+
+          raise ArgumentError, "No URL configured for queue #{queue}"
+        end
+
+        def poller_options(queue)
+          config = Aws::ActiveJob::SQS.config
+          queue_url = config.url_for(queue)
           poller_options = {
             skip_delete: true,
-            max_number_of_messages: @options[:max_messages],
-            visibility_timeout: @options[:visibility_timeout]
+            max_number_of_messages: config.max_messages_for(queue),
+            visibility_timeout: config.visibility_timeout_for(queue)
           }
+
           # Limit max_number_of_messages for FIFO queues to 1
           # this ensures jobs with the same message_group_id are processed
           # in order
           # Jobs with different message_group_id will be processed in
           # parallel and may be out of order.
           poller_options[:max_number_of_messages] = 1 if Aws::ActiveJob::SQS.fifo?(queue_url)
+          poller_options
+        end
 
+        def _poll(poller_options, queue_url)
+          poller = Aws::SQS::QueuePoller.new(
+            queue_url,
+            client: Aws::ActiveJob::SQS.config.client
+          )
           single_message = poller_options[:max_number_of_messages] == 1
-
-          @poller.poll(poller_options) do |msgs|
+          poller.poll(poller_options) do |msgs|
             msgs = [msgs] if single_message
-            @logger.info "Processing batch of #{msgs.length} messages"
-            msgs.each do |msg|
-              @executor.execute(Aws::SQS::Message.new(
-                                  queue_url: queue_url,
-                                  receipt_handle: msg.receipt_handle,
-                                  data: msg,
-                                  client: client
-                                ))
-            end
+            execute_messages(msgs, queue_url)
           end
         end
 
-        def boot_rails
-          ENV['RACK_ENV'] = ENV['RAILS_ENV'] = @environment
-          require 'rails'
-          require File.expand_path('config/environment.rb')
-        end
-
-        # rubocop:disable Metrics
-        def parse_args(argv)
-          out = {}
-          parser = ::OptionParser.new do |opts|
-            opts.on('-q', '--queue STRING', '[Required] Queue to poll') { |a| out[:queue] = a }
-            opts.on('-e', '--environment STRING',
-                    'Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.') do |a|
-              out[:environment] = a
-            end
-            opts.on('-t', '--threads INTEGER', Integer,
-                    'The maximum number of worker threads to create.  Defaults to 2x the number of processors available on this system.') do |a|
-              out[:threads] = a
-            end
-            opts.on('-b', '--backpressure INTEGER', Integer,
-                    'The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.') do |a|
-              out[:backpressure] = a
-            end
-            opts.on('-m', '--max_messages INTEGER', Integer,
-                    'Max number of messages to receive in a batch from SQS.') do |a|
-              out[:max_messages] = a
-            end
-            opts.on('-v', '--visibility_timeout INTEGER', Integer,
-                    'The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.') do |a|
-              out[:visibility_timeout] = a
-            end
-            opts.on('-s', '--shutdown_timeout INTEGER', Integer,
-                    'The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.') do |a|
-              out[:shutdown_timeout] = a
-            end
-            opts.on('--[no-]retry_standard_errors [FLAG]', TrueClass,
-                    'When set, retry all StandardErrors (leaving failed messages on the SQS Queue). These retries are ON TOP of standard Rails ActiveJob retries set by retry_on in the ActiveJob.') do |a|
-              out[:retry_standard_errors] = a.nil? ? true : a
-            end
+        def execute_messages(msgs, queue_url)
+          @logger.info "Processing batch of #{msgs.length} messages"
+          msgs.each do |msg|
+            sqs_message = Aws::SQS::Message.new(
+              queue_url: queue_url,
+              receipt_handle: msg.receipt_handle,
+              data: msg,
+              client: Aws::ActiveJob::SQS.config.client
+            )
+            @executor.execute(sqs_message)
           end
-
-          parser.banner = 'aws_sqs_active_job [options]'
-          parser.on_tail '-h', '--help', 'Show help' do
-            puts parser
-            exit 1
-          end
-
-          parser.parse(argv)
-          out
-        end
-        # rubocop:enable Metrics
-
-        def validate_config
-          raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]
         end
       end
     end

data/lib/aws-activejob-sqs.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'active_job'
 require_relative 'active_job/queue_adapters/sqs_adapter'
 require_relative 'active_job/queue_adapters/sqs_adapter/params'
 require_relative 'active_job/queue_adapters/sqs_async_adapter'
@@ -11,6 +12,7 @@ require_relative 'aws/active_job/sqs/lambda_handler'
 
 module Aws
   module ActiveJob
+    # ActiveJob Adapter and backend queueing using AWS SQS.
     module SQS
       VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
 
@@ -19,7 +21,7 @@ module Aws
         @config ||= Configuration.new
       end
 
-      # @yield Configuration
+      # @yield [Configuration] the (singleton) Configuration
       def self.configure
         yield(config)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-activejob-sqs
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-02 00:00:00.000000000 Z
+date: 2024-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -62,18 +62,19 @@ description: Amazon Simple Queue Service as an ActiveJob adapter
 email:
 - aws-dr-rubygems@amazon.com
 executables:
-- aws_sqs_active_job
+- aws_active_job_sqs
 extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE
 - VERSION
-- bin/aws_sqs_active_job
+- bin/aws_active_job_sqs
 - lib/active_job/queue_adapters/sqs_adapter.rb
 - lib/active_job/queue_adapters/sqs_adapter/params.rb
 - lib/active_job/queue_adapters/sqs_async_adapter.rb
 - lib/aws-activejob-sqs.rb
+- lib/aws/active_job/sqs/cli_options.rb
 - lib/aws/active_job/sqs/configuration.rb
 - lib/aws/active_job/sqs/deduplication.rb
 - lib/aws/active_job/sqs/executor.rb

data/bin/aws_sqs_active_job

@@ -1,9 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative '../lib/aws/active_job/sqs/poller'
-
-require 'rails'
-require File.expand_path("config/environment.rb")
-
-Aws::ActiveJob::SQS::Poller.new(ARGV).run