shoryuken 7.0.2 → 7.0.3

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -48,8 +48,17 @@ module Shoryuken
           # @return [Shoryuken::Helpers::TimerTask] the timer task
           def auto_extend(_worker, queue, sqs_msg, _body)
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
+            execution_interval = extension_interval(queue_visibility_timeout)
 
-            Shoryuken::Helpers::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
+            unless execution_interval
+              logger.warn do
+                "Not auto-extending visibility for #{queue}/#{sqs_msg.message_id}: queue visibility " \
+                "timeout (#{queue_visibility_timeout}s) is too short. Increase it to use auto_visibility_timeout."
+              end
+              return nil
+            end
+
+            Shoryuken::Helpers::TimerTask.new(execution_interval: execution_interval) do
               logger.debug do
                 "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
               end
@@ -61,6 +70,25 @@ module Shoryuken
               end
             end
           end
+
+          private
+
+          # Returns a positive interval at which to re-extend the message's
+          # visibility before it expires, or nil when the visibility timeout is
+          # too short to schedule one. Normally this is EXTEND_UPFRONT_SECONDS
+          # before expiry, but for short timeouts (<= EXTEND_UPFRONT_SECONDS) it
+          # falls back to half the timeout so the timer still fires in time
+          # instead of TimerTask raising on a non-positive interval.
+          #
+          # @param visibility_timeout [Integer] the queue's visibility timeout in seconds
+          # @return [Float, Integer, nil] the execution interval, or nil if too short
+          def extension_interval(visibility_timeout)
+            return nil if visibility_timeout <= 0
+
+            interval = visibility_timeout - EXTEND_UPFRONT_SECONDS
+            interval = visibility_timeout / 2.0 if interval <= 0
+            interval
+          end
         end
 
         # Creates and starts a visibility extension timer
@@ -71,7 +99,9 @@ module Shoryuken
         # @param body [Object] the parsed message body
         # @return [Shoryuken::Helpers::TimerTask] the started timer
         def auto_visibility_timer(worker, queue, sqs_msg, body)
-          MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body).tap(&:execute)
+          timer = MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body)
+          timer&.execute
+          timer
         end
       end
     end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -17,8 +17,8 @@ module Shoryuken
         # @param _body [Object] the parsed message body (unused)
         # @yield continues to the next middleware in the chain
         # @return [void]
-        # @raise [StandardError] re-raises the original exception if retry intervals are not configured
-        #   or if retry limit is exceeded
+        # @raise [StandardError] re-raises the original exception if retry intervals are not configured,
+        #   if retry limit is exceeded, or if the exception is classified as non-retryable
         def call(worker, _queue, sqs_msg, _body)
           return yield unless worker.class.exponential_backoff?
 
@@ -30,7 +30,12 @@ module Shoryuken
           started_at = Time.now
           yield
         rescue => e
-          retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
+          worker_options = worker.class.get_shoryuken_options
+          retry_intervals = worker_options['retry_intervals']
+
+          # Non-retryable exceptions must not be backoff retried; re-raise so the
+          # NonRetryableException middleware can delete the message immediately.
+          raise if NonRetryableException.non_retryable?(e, worker_options['non_retryable_exceptions'])
 
           if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
             # Re-raise the exception if the job is not going to be exponential backoff retried.

data/lib/shoryuken/middleware/server/non_retryable_exception.rb

@@ -28,6 +28,22 @@ module Shoryuken
       class NonRetryableException
         include Util
 
+        # Checks whether an exception is classified as non-retryable for a worker
+        #
+        # @param exception [Exception] the exception to classify
+        # @param non_retryable_exceptions [Array<Class>, #call, nil] the worker's
+        #   non_retryable_exceptions option: exception classes or a callable
+        # @return [Boolean] true if the exception must not be retried
+        def self.non_retryable?(exception, non_retryable_exceptions)
+          return false unless non_retryable_exceptions
+
+          if non_retryable_exceptions.respond_to?(:call)
+            !!non_retryable_exceptions.call(exception)
+          else
+            Array(non_retryable_exceptions).any? { |klass| exception.is_a?(klass) }
+          end
+        end
+
         # Processes a message and handles non-retryable exceptions
         #
         # @param worker [Object] the worker instance
@@ -41,14 +57,7 @@ module Shoryuken
         rescue => e
           non_retryable_exceptions = worker.class.get_shoryuken_options['non_retryable_exceptions']
 
-          return raise unless non_retryable_exceptions
-
-          if non_retryable_exceptions.respond_to?(:call)
-            return raise unless non_retryable_exceptions.call(e)
-          else
-            exception_classes = Array(non_retryable_exceptions)
-            return raise unless exception_classes.any? { |klass| e.is_a?(klass) }
-          end
+          return raise unless self.class.non_retryable?(e, non_retryable_exceptions)
 
           # Handle batch messages
           messages = sqs_msg.is_a?(Array) ? sqs_msg : [sqs_msg]

data/lib/shoryuken/options.rb

@@ -34,7 +34,8 @@ module Shoryuken
     # @return [Class] the executor class for running workers
     # @return [Shoryuken::WorkerRegistry] the registry for worker classes
     # @return [Array<#call>] handlers for processing exceptions
-    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout,
+    attr_accessor :active_job_queue_name_prefixing, :active_job_fifo_message_deduplication,
+                  :cache_visibility_timeout,
                   :groups, :launcher_executor, :reloader, :enable_reloading,
                   :start_callback, :stop_callback, :worker_executor, :worker_registry,
                   :exception_handlers
@@ -53,6 +54,7 @@ module Shoryuken
       self.worker_registry = DefaultWorkerRegistry.new
       self.exception_handlers = [DefaultExceptionHandler]
       self.active_job_queue_name_prefixing = false
+      self.active_job_fifo_message_deduplication = true
       self.worker_executor = Worker::DefaultExecutor
       self.cache_visibility_timeout = false
       self.reloader = proc { |&block| block.call }
@@ -302,6 +304,15 @@ module Shoryuken
       @active_job_queue_name_prefixing
     end
 
+    # Checks if the ActiveJob adapter should auto-generate a content-based
+    # message_deduplication_id for FIFO queues. When disabled, distinct enqueues
+    # of the same job class and arguments are no longer silently deduplicated.
+    #
+    # @return [Boolean] true if FIFO deduplication id generation is enabled
+    def active_job_fifo_message_deduplication?
+      @active_job_fifo_message_deduplication
+    end
+
     private
 
     # Creates the default server middleware chain

data/lib/shoryuken/polling/strict_priority.rb

@@ -22,6 +22,10 @@ module Shoryuken
                         .each_with_object({}) { |queue, h| h[queue] = Time.at(0) }
 
         @delay = delay
+        # next_queue/messages_found run on the dispatch thread while
+        # message_processed runs on processor-completion threads (for FIFO
+        # queues), so all access to the shared state is serialized.
+        @mutex = Mutex.new
         # Start queues at 0
         reset_next_queue
       end
@@ -30,8 +34,10 @@ module Shoryuken
       #
       # @return [QueueConfiguration, nil] the next queue configuration or nil if all paused
       def next_queue
-        next_queue = next_active_queue
-        next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
+        @mutex.synchronize do
+          next_queue = next_active_queue
+          next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
+        end
       end
 
       # Handles the result of polling a queue
@@ -40,10 +46,12 @@ module Shoryuken
       # @param messages_found [Integer] number of messages found
       # @return [void]
       def messages_found(queue, messages_found)
-        if messages_found == 0
-          pause(queue)
-        else
-          reset_next_queue
+        @mutex.synchronize do
+          if messages_found == 0
+            pause(queue)
+          else
+            reset_next_queue
+          end
         end
       end
 
@@ -51,11 +59,13 @@ module Shoryuken
       #
       # @return [Array<Array>] array of [queue_name, priority] pairs
       def active_queues
-        @queues
-          .reverse
-          .map.with_index(1)
-          .reject { |q, _| queue_paused?(q) }
-          .reverse
+        @mutex.synchronize do
+          @queues
+            .reverse
+            .map.with_index(1)
+            .reject { |q, _| queue_paused?(q) }
+            .reverse
+        end
       end
 
       # Called when a message from a queue has been processed
@@ -63,9 +73,11 @@ module Shoryuken
       # @param queue [String] the queue name
       # @return [void]
       def message_processed(queue)
-        if queue_paused?(queue)
-          logger.debug "Unpausing #{queue}"
-          @paused_until[queue] = Time.at 0
+        @mutex.synchronize do
+          if queue_paused?(queue)
+            logger.debug "Unpausing #{queue}"
+            @paused_until[queue] = Time.at(0)
+          end
         end
       end
 

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -15,18 +15,26 @@ module Shoryuken
         @queues = queues.dup.uniq
         @paused_queues = []
         @delay = delay
+        # next_queue/messages_found run on the dispatch thread while
+        # message_processed runs on processor-completion threads (for FIFO
+        # queues), so all access to the shared state is serialized.
+        @mutex = Mutex.new
       end
 
       # Returns the next queue to poll in round-robin order
       #
       # @return [QueueConfiguration, nil] the next queue configuration or nil if all paused
       def next_queue
-        unpause_queues
-        queue = @queues.shift
-        return nil if queue.nil?
-
-        @queues << queue
-        QueueConfiguration.new(queue, {})
+        @mutex.synchronize do
+          unpause_queues
+          queue = @queues.shift
+          if queue.nil?
+            nil
+          else
+            @queues << queue
+            QueueConfiguration.new(queue, {})
+          end
+        end
       end
 
       # Handles the result of polling a queue, adjusting weight if messages were found
@@ -35,16 +43,17 @@ module Shoryuken
       # @param messages_found [Integer] number of messages found
       # @return [void]
       def messages_found(queue, messages_found)
-        if messages_found == 0
-          pause(queue)
-          return
-        end
-
-        maximum_weight = maximum_queue_weight(queue)
-        current_weight = current_queue_weight(queue)
-        if maximum_weight > current_weight
-          logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
-          @queues << queue
+        @mutex.synchronize do
+          if messages_found == 0
+            pause(queue)
+          else
+            maximum_weight = maximum_queue_weight(queue)
+            current_weight = current_queue_weight(queue)
+            if maximum_weight > current_weight
+              logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
+              @queues << queue
+            end
+          end
         end
       end
 
@@ -52,7 +61,7 @@ module Shoryuken
       #
       # @return [Array<Array>] array of [queue_name, weight] pairs
       def active_queues
-        unparse_queues(@queues)
+        @mutex.synchronize { unparse_queues(@queues) }
       end
 
       # Called when a message from a queue has been processed
@@ -60,10 +69,10 @@ module Shoryuken
       # @param queue [String] the queue name
       # @return [void]
       def message_processed(queue)
-        paused_queue = @paused_queues.find { |_time, name| name == queue }
-        return unless paused_queue
-
-        paused_queue[0] = Time.at 0
+        @mutex.synchronize do
+          paused_queue = @paused_queues.find { |_time, name| name == queue }
+          paused_queue[0] = Time.at(0) if paused_queue
+        end
       end
 
       private
@@ -79,16 +88,21 @@ module Shoryuken
         logger.debug "Paused #{queue}"
       end
 
-      # Unpauses queues whose delay has expired
+      # Unpauses the first queue whose delay has expired.
+      #
+      # Scans for any expired entry rather than only the head of the list:
+      # message_processed marks a queue ready by setting its time to the epoch,
+      # and that entry may sit behind an earlier-paused queue that is still
+      # paused - checking only the head would leave the ready queue stuck.
       #
       # @return [void]
       def unpause_queues
-        return if @paused_queues.empty?
-        return if Time.now < @paused_queues.first[0]
+        index = @paused_queues.index { |time, _name| time <= Time.now }
+        return if index.nil?
 
-        pause = @paused_queues.shift
-        @queues << pause[1]
-        logger.debug "Unpaused #{pause[1]}"
+        paused = @paused_queues.delete_at(index)
+        @queues << paused[1]
+        logger.debug "Unpaused #{paused[1]}"
       end
 
       # Returns the current weight of a queue in the active rotation

data/lib/shoryuken/queue.rb

@@ -52,11 +52,18 @@ module Shoryuken
       failed_messages = client.delete_message_batch(
         options.merge(queue_url: url)
       ).failed || []
-      failed_messages.any? do |failure|
+
+      # Log every failure (not just the first) and return a plain boolean.
+      # The previous `any? { logger.error ... }` short-circuited after the first
+      # failure - so the rest went unlogged - and only returned truthy because
+      # Logger#error happens to return true.
+      failed_messages.each do |failure|
         logger.error do
           "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
         end
       end
+
+      failed_messages.any?
     end
 
     # Sends a single message to the queue

data/lib/shoryuken/util.rb

@@ -20,7 +20,10 @@ module Shoryuken
     def fire_event(event, reverse = false, event_options = {})
       logger.debug { "Firing '#{event}' lifecycle event" }
       arr = Shoryuken.options[:lifecycle_events][event]
-      arr.reverse! if reverse
+      # reverse (not reverse!) so the stored handler array keeps its original
+      # order: events fired more than once with reverse: true (e.g. :shutdown
+      # via stop then stop!) must not alternate their handler order.
+      arr = arr.reverse if reverse
       arr.each do |block|
         block.call(event_options)
       rescue => e

data/lib/shoryuken/version.rb

@@ -2,5 +2,5 @@
 
 module Shoryuken
   # Current version of the Shoryuken gem
-  VERSION = '7.0.2'
+  VERSION = '7.0.3'
 end

data/lib/shoryuken/worker.rb

@@ -224,7 +224,11 @@ module Shoryuken
       #
       # @example Configuring exponential backoff
       #   shoryuken_options retry_intervals: [1, 5, 25, 125, 625]
-      #   # Will retry after 1s, 5s, 25s, 125s, then 625s before giving up
+      #   # Retries after 1s, 5s, 25s, 125s, then 625s for every later attempt.
+      #   # Shoryuken does not stop retrying on its own once the intervals are
+      #   # exhausted - it keeps reusing the last interval. Configure an SQS
+      #   # redrive policy (maxReceiveCount) to send exhausted messages to a
+      #   # dead-letter queue.
       #
       # @see #shoryuken_options Documentation for configuring retry_intervals
       def exponential_backoff?

data/lib/shoryuken.rb

@@ -58,6 +58,8 @@ module Shoryuken
     :stop_callback=,
     :active_job_queue_name_prefixing?,
     :active_job_queue_name_prefixing=,
+    :active_job_fifo_message_deduplication?,
+    :active_job_fifo_message_deduplication=,
     :sqs_client,
     :sqs_client=,
     :sqs_client_receive_message_opts,

data/renovate.json

@@ -56,7 +56,21 @@
         "ruby/setup-ruby",
         "ruby"
       ],
-      "groupName": "ruby setup"
+      "groupName": "ruby setup",
+      "internalChecksFilter": "strict"
+    },
+    {
+      "description": "Let setup-ruby pass age gate before ruby so it is ready when the group PR is created",
+      "matchPackageNames": [
+        "ruby/setup-ruby"
+      ],
+      "minimumReleaseAge": "5 days"
     }
-  ]
+  ],
+  "lockFileMaintenance": {
+    "enabled": true,
+    "schedule": [
+      "before 4am on the first day of the month"
+    ]
+  }
 }

data/spec/integration/active_job/current_attributes/cross_job_reset_spec.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# A message without a CurrentAttributes key must not inherit a previous job's
+# context. Every registered class is reset after each job - even keyless ones -
+# so values set during one job don't leak into the next on a reused worker
+# thread.
+
+setup_sqs
+setup_active_job
+
+require 'active_support/current_attributes'
+require 'shoryuken/active_job/current_attributes'
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+
+class LeakCurrent < ActiveSupport::CurrentAttributes
+  attribute :user_id
+end
+
+Shoryuken::ActiveJob::CurrentAttributes.persist(LeakCurrent)
+
+class CrossJobResetTestJob < ActiveJob::Base
+  def perform
+    # Record what the worker thread sees at the start of the job, then dirty
+    # Current. Without a reset after a keyless job, the second execution on the
+    # same thread would observe 'leaked' instead of nil.
+    DT[:observed] << LeakCurrent.user_id
+    LeakCurrent.user_id = 'leaked'
+  end
+end
+
+CrossJobResetTestJob.queue_as(queue_name)
+
+# Concurrency 1 so the two jobs run sequentially on the same worker thread.
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
+
+# Enqueued with an empty context, so both messages carry no cattr key.
+CrossJobResetTestJob.perform_later
+CrossJobResetTestJob.perform_later
+
+poll_queues_until(timeout: 30) { DT[:observed].size >= 2 }
+
+assert_equal([nil, nil], DT[:observed].first(2),
+             "CurrentAttributes leaked across jobs: #{DT[:observed].inspect}")

data/spec/integration/active_job/current_attributes/incremental_persist_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# CurrentAttributes registered one class per persist call are all preserved.
+#
+# persist derived each storage key from the per-call index, so calling it once
+# per class (as an app might across several initializers) made the third call
+# reuse "cattr_0" and silently overwrite the second class - its attributes were
+# then never serialized or restored.
+
+setup_sqs
+setup_active_job
+
+require 'active_support/current_attributes'
+require 'shoryuken/active_job/current_attributes'
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+
+class CurrentA < ActiveSupport::CurrentAttributes
+  attribute :a_val
+end
+
+class CurrentB < ActiveSupport::CurrentAttributes
+  attribute :b_val
+end
+
+class CurrentC < ActiveSupport::CurrentAttributes
+  attribute :c_val
+end
+
+# Registered incrementally, one class per call.
+Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentA)
+Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentB)
+Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentC)
+
+# Every class must be registered, each under a distinct storage key.
+registered = Shoryuken::ActiveJob::CurrentAttributes.cattrs
+assert_equal(
+  %w[CurrentA CurrentB CurrentC],
+  registered.values.sort,
+  'every persisted CurrentAttributes class must be registered (none silently dropped)'
+)
+assert_equal(3, registered.keys.uniq.size, 'each class must use a distinct storage key')
+
+class IncrementalCurrentJob < ActiveJob::Base
+  def perform
+    DT[:executions] << {
+      a_val: CurrentA.a_val,
+      b_val: CurrentB.b_val,
+      c_val: CurrentC.c_val
+    }
+  end
+end
+
+IncrementalCurrentJob.queue_as(queue_name)
+
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
+
+CurrentA.a_val = 'a-value'
+CurrentB.b_val = 'b-value'
+CurrentC.c_val = 'c-value'
+
+IncrementalCurrentJob.perform_later
+
+CurrentA.reset
+CurrentB.reset
+CurrentC.reset
+
+poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
+
+result = DT[:executions].first
+assert_equal('a-value', result[:a_val])
+assert_equal('b-value', result[:b_val], 'the second incrementally-persisted class must not be dropped')
+assert_equal('c-value', result[:c_val])

data/spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+# This spec tests opting out of the automatic FIFO message_deduplication_id that
+# the ActiveJob adapter generates.
+#
+# For FIFO queues the adapter derives a content-based message_deduplication_id
+# from the serialized job minus job_id/enqueued_at (see issues #457 / #750). That
+# means two *distinct* enqueues of the same job class + args within SQS's 5-minute
+# window collapse into one - the second is silently dropped. That is intentional,
+# but it is a "skipped message" trap for users who legitimately enqueue the same
+# work twice.
+#
+# Expected behavior: setting Shoryuken.active_job_fifo_message_deduplication = false
+# stops the adapter from generating the id, so those enqueues are no longer
+# silently deduplicated. The default (true) preserves the existing behavior.
+#
+# Regression: there was no way to disable the auto-generated deduplication id.
+
+setup_active_job
+
+class FifoOptOutJob < ActiveJob::Base
+  queue_as :fifo_opt_out
+
+  def perform(_arg); end
+end
+
+fifo_queue_mock = Object.new
+fifo_queue_mock.define_singleton_method(:fifo?) { true }
+fifo_queue_mock.define_singleton_method(:name) { 'fifo_opt_out.fifo' }
+
+captured = nil
+fifo_queue_mock.define_singleton_method(:send_message) { |params| captured = params }
+
+Shoryuken::Client.define_singleton_method(:queues) { |_queue_name = nil| fifo_queue_mock }
+Shoryuken.define_singleton_method(:register_worker) { |*| nil }
+
+# Default behavior: the adapter sets a content-based dedup id (excluding the
+# per-enqueue job_id/enqueued_at), so two identical jobs would collapse to one.
+FifoOptOutJob.perform_later('same-args')
+
+assert(
+  captured.key?(:message_deduplication_id),
+  'by default a FIFO ActiveJob send gets an auto-generated message_deduplication_id'
+)
+
+body = captured[:message_body]
+expected = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id', 'enqueued_at')))
+assert_equal(expected, captured[:message_deduplication_id])
+
+# Opt-out: with deduplication disabled the adapter no longer sets a dedup id, so
+# distinct enqueues of identical jobs are not silently dropped.
+Shoryuken.active_job_fifo_message_deduplication = false
+captured = nil
+FifoOptOutJob.perform_later('same-args')
+
+refute(
+  captured.key?(:message_deduplication_id),
+  'with deduplication disabled the adapter must not set message_deduplication_id'
+)
+
+# An explicit deduplication id must still be honored even when auto-generation is off.
+captured = nil
+FifoOptOutJob.set(message_deduplication_id: 'explicit-id').perform_later('same-args')
+
+assert_equal('explicit-id', captured[:message_deduplication_id])