shoryuken 5.3.0 → 6.2.1

data/README.md

@@ -1,4 +1,4 @@
-**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? Fill up this form https://forms.gle/8kTso8ixa9Sfp6rJ9**
+**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? [Join our Slack](https://join.slack.com/t/shoryuken/shared_invite/zt-19xjq3iqc-KmoJ6eU6~qvZNqcLzIrjww)**
 
 # Shoryuken
 
@@ -86,3 +86,21 @@ To run integration specs, start a mock SQS server on `localhost:5000`. One such
 ```sh
 bundle exec rake spec:integration
 ```
+
+### To release a new version
+
+Compare latest tag with HEAD:
+
+```sh
+git log $(git describe --tags --abbrev=0)..HEAD --oneline
+```
+
+then update CHANGELOG.md.
+
+Update version in `lib/shoryuken/version.rb` with the appropriate version number [SEMVER](https://semver.org/).
+
+then run:
+
+```sh
+bundle exec rake release
+```

data/bin/cli/sqs.rb

@@ -117,7 +117,7 @@ module Shoryuken
               max_number_of_messages: batch_size,
               attribute_names: ['All'],
               message_attribute_names: ['All']
-            ).messages
+            ).messages || []
 
             messages.each { |m| yield m }
 

data/gemfiles/aws_sdk_core_2.gemfile

@@ -14,7 +14,7 @@ end
 
 group :development do
   gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
-  gem "pry-byebug", "3.9.0"
+  gem "pry-byebug"
   gem "rubocop"
 end
 

data/gemfiles/rails_7_0.gemfile

@@ -0,0 +1,22 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 7.0"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "rubocop"
+  gem "pry", ">= 0.14.2"
+  gem "pry-byebug", ">= 3.10.1"
+end
+
+gemspec path: "../"

data/lib/shoryuken/default_exception_handler.rb

@@ -0,0 +1,10 @@
+module Shoryuken
+  class DefaultExceptionHandler
+    extend Util
+
+    def self.call(exception, _queue, _sqs_msg)
+      logger.error { "Processor failed: #{exception.message}" }
+      logger.error { exception.backtrace.join("\n") } if exception.backtrace
+    end
+  end
+end

data/lib/shoryuken/environment_loader.rb

@@ -18,12 +18,12 @@ module Shoryuken
     end
 
     def setup_options
+      initialize_rails if load_rails?
       initialize_options
       initialize_logger
     end
 
     def load
-      load_rails if Shoryuken.options[:rails]
       prefix_active_job_queue_names
       parse_queues
       require_workers
@@ -55,7 +55,7 @@ module Shoryuken
       Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
-    def load_rails
+    def initialize_rails
       # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
 
       require 'rails'
@@ -70,6 +70,13 @@ module Shoryuken
             ::Rails.application.config.eager_load = true
           end
         end
+        ::Rails::Application.initializer 'shoryuken.set_reloader_hook' do |app|
+          Shoryuken.reloader = proc do |&block|
+            app.reloader.wrap do
+              block.call
+            end
+          end
+        end
         if Shoryuken.active_job?
           require 'shoryuken/extensions/active_job_extensions'
           require 'shoryuken/extensions/active_job_adapter'
@@ -79,6 +86,10 @@ module Shoryuken
       end
     end
 
+    def load_rails?
+      options[:rails]
+    end
+
     def prefix_active_job_queue_name(queue_name, weight)
       return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
 
@@ -159,9 +170,17 @@ module Shoryuken
 
       return if non_existent_queues.none?
 
+      # NOTE: HEREDOC's ~ operator removes indents, but is only available Ruby 2.3+
+      # See github PR: https://github.com/ruby-shoryuken/shoryuken/pull/691#issuecomment-1007653595
+      error_msg = <<-MSG.gsub(/^\s+/, '')
+        The specified queue(s) #{non_existent_queues.join(', ')} do not exist.
+        Try 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings.
+        It's also possible that you don't have permission to access the specified queues.
+      MSG
+
       fail(
         ArgumentError,
-        "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nTry 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
+        error_msg
       )
     end
 

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -66,8 +66,11 @@ module ActiveJob
         }
 
         if queue.fifo?
-          # See https://github.com/phstc/shoryuken/issues/457
-          msg[:message_deduplication_id] = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id')))
+          # See https://github.com/ruby-shoryuken/shoryuken/issues/457 and
+          # https://github.com/ruby-shoryuken/shoryuken/pull/750#issuecomment-1781317929
+          msg[:message_deduplication_id] = Digest::SHA256.hexdigest(
+            JSON.dump(body.except('job_id', 'enqueued_at'))
+          )
         end
 
         msg.merge(job_params.except(:message_attributes))

data/lib/shoryuken/extensions/active_job_extensions.rb

@@ -11,7 +11,7 @@ module Shoryuken
       end
     end
 
-    # Initializes SQS SendMessage parameters on instances of ActiveJobe::Base
+    # Initializes SQS SendMessage parameters on instances of ActiveJob::Base
     # to the empty hash, and populates it whenever `#enqueue` is called, such
     # as when using ActiveJob::Base.set.
     module SQSSendMessageParametersSupport

data/lib/shoryuken/launcher.rb

@@ -16,11 +16,13 @@ module Shoryuken
     def stop!
       initiate_stop
 
-      executor.shutdown
+      # Don't await here so the timeout below is not delayed
+      stop_new_dispatching
 
-      return if executor.wait_for_termination(Shoryuken.options[:timeout])
+      executor.shutdown
+      executor.kill unless executor.wait_for_termination(Shoryuken.options[:timeout])
 
-      executor.kill
+      fire_event(:stopped)
     end
 
     def stop
@@ -28,8 +30,13 @@ module Shoryuken
 
       initiate_stop
 
+      stop_new_dispatching
+      await_dispatching_in_progress
+
       executor.shutdown
       executor.wait_for_termination
+
+      fire_event(:stopped)
     end
 
     def healthy?
@@ -41,6 +48,14 @@ module Shoryuken
 
     private
 
+    def stop_new_dispatching
+      @managers.each(&:stop_new_dispatching)
+    end
+
+    def await_dispatching_in_progress
+      @managers.each(&:await_dispatching_in_progress)
+    end
+
     def executor
       @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
     end

data/lib/shoryuken/logging.rb

@@ -30,11 +30,11 @@ module Shoryuken
     end
 
     def self.logger
-      @logger || initialize_logger
+      @logger ||= initialize_logger
     end
 
     def self.logger=(log)
-      @logger = (log ? log : Logger.new('/dev/null'))
+      @logger = (log || Logger.new('/dev/null'))
     end
   end
 end

data/lib/shoryuken/manager.rb

@@ -9,13 +9,15 @@ module Shoryuken
     attr_reader :group
 
     def initialize(group, fetcher, polling_strategy, concurrency, executor)
-      @group            = group
-      @fetcher          = fetcher
-      @polling_strategy = polling_strategy
-      @max_processors   = concurrency
-      @busy_processors  = Concurrent::AtomicFixnum.new(0)
-      @executor         = executor
-      @running          = Concurrent::AtomicBoolean.new(true)
+      @group                      = group
+      @fetcher                    = fetcher
+      @polling_strategy           = polling_strategy
+      @max_processors             = concurrency
+      @busy_processors            = Concurrent::AtomicFixnum.new(0)
+      @executor                   = executor
+      @running                    = Concurrent::AtomicBoolean.new(true)
+      @stop_new_dispatching       = Concurrent::AtomicBoolean.new(false)
+      @dispatching_release_signal = ::Queue.new
     end
 
     def start
@@ -23,6 +25,17 @@ module Shoryuken
       dispatch_loop
     end
 
+    def stop_new_dispatching
+      @stop_new_dispatching.make_true
+    end
+
+    def await_dispatching_in_progress
+      # There might still be a dispatching on-going, as the response from SQS could take some time
+      # We don't want to stop the process before processing incoming messages, as they would stay "in-flight" for some time on SQS
+      # We use a queue, as the dispatch_loop is running on another thread, and this is a efficient way of communicating between threads.
+      @dispatching_release_signal.pop
+    end
+
     def running?
       @running.true? && @executor.running?
     end
@@ -30,7 +43,10 @@ module Shoryuken
     private
 
     def dispatch_loop
-      return unless running?
+      if @stop_new_dispatching.true? || !running?
+        @dispatching_release_signal << 1
+        return
+      end
 
       @executor.post { dispatch }
     end
@@ -94,7 +110,6 @@ module Shoryuken
 
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
-
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end

data/lib/shoryuken/middleware/server/active_record.rb

@@ -5,7 +5,11 @@ module Shoryuken
         def call(*_args)
           yield
         ensure
-          ::ActiveRecord::Base.clear_active_connections!
+          if ::ActiveRecord.version >= Gem::Version.new('7.1')
+            ::ActiveRecord::Base.connection_handler.clear_active_connections!(:all)
+          else
+            ::ActiveRecord::Base.clear_active_connections!
+          end
         end
       end
     end

data/lib/shoryuken/options.rb

@@ -11,22 +11,26 @@ module Shoryuken
         dispatch: [],
         utilization_update: [],
         quiet: [],
-        shutdown: []
+        shutdown: [],
+        stopped: []
       }
     }.freeze
 
     attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
-                  :launcher_executor,
-                  :start_callback, :stop_callback, :worker_executor, :worker_registry
+                  :launcher_executor, :reloader, :enable_reloading,
+                  :start_callback, :stop_callback, :worker_executor, :worker_registry, :exception_handlers
     attr_writer :default_worker_options, :sqs_client
     attr_reader :sqs_client_receive_message_opts
 
     def initialize
       self.groups = {}
       self.worker_registry = DefaultWorkerRegistry.new
+      self.exception_handlers = [DefaultExceptionHandler]
       self.active_job_queue_name_prefixing = false
       self.worker_executor = Worker::DefaultExecutor
       self.cache_visibility_timeout = false
+      self.reloader = proc { |&block| block.call }
+      self.enable_reloading ||= false
       # this is needed for keeping backward compatibility
       @sqs_client_receive_message_opts ||= {}
     end
@@ -63,10 +67,14 @@ module Shoryuken
         Polling::WeightedRoundRobin
       when 'StrictPriority'
         Polling::StrictPriority
+      when String
+        begin
+          Object.const_get(strategy)
+        rescue NameError
+          raise ArgumentError, "#{strategy} is not a valid polling_strategy"
+        end
       when Class
         strategy
-      else
-        raise ArgumentError, "#{strategy} is not a valid polling_strategy"
       end
     end
 
@@ -134,7 +142,7 @@ module Shoryuken
     end
 
     # Register a block to run at a point in the Shoryuken lifecycle.
-    # :startup, :quiet or :shutdown are valid events.
+    # :startup, :quiet, :shutdown or :stopped are valid events.
     #
     #   Shoryuken.configure_server do |config|
     #     config.on(:shutdown) do

data/lib/shoryuken/polling/strict_priority.rb

@@ -39,8 +39,10 @@ module Shoryuken
       end
 
       def message_processed(queue)
-        logger.debug "Unpausing #{queue}"
-        @paused_until[queue] = Time.now
+        if queue_paused?(queue)
+          logger.debug "Unpausing #{queue}"
+          @paused_until[queue] = Time.at 0
+        end
       end
 
       private

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -36,12 +36,10 @@ module Shoryuken
       end
 
       def message_processed(queue)
-        return if @paused_queues.empty?
+        paused_queue = @paused_queues.find { |_time, name| name == queue }
+        return unless paused_queue
 
-        logger.debug "Unpausing #{queue}"
-        @paused_queues.reject! { |_time, name| name == queue }
-        @queues << queue
-        @queues.uniq!
+        paused_queue[0] = Time.at 0
       end
 
       private

data/lib/shoryuken/processor.rb

@@ -14,16 +14,24 @@ module Shoryuken
     end
 
     def process
-      return logger.error { "No worker found for #{queue}" } unless worker
+      worker_perform = proc do
+        return logger.error { "No worker found for #{queue}" } unless worker
+        Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
+          worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
+            worker.perform(sqs_msg, body)
+          end
+        end
+      end
 
-      Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
-        worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
-          worker.perform(sqs_msg, body)
+      if Shoryuken.enable_reloading
+        Shoryuken.reloader.call do
+          worker_perform.call
         end
+      else
+        worker_perform.call
       end
     rescue Exception => ex
-      logger.error { "Processor failed: #{ex.message}" }
-      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+      Array(Shoryuken.exception_handlers).each { |handler| handler.call(ex, queue, sqs_msg) }
 
       raise
     end

data/lib/shoryuken/queue.rb

@@ -21,9 +21,10 @@ module Shoryuken
     end
 
     def delete_messages(options)
-      client.delete_message_batch(
+      failed_messages = client.delete_message_batch(
         options.merge(queue_url: url)
-      ).failed.any? do |failure|
+      ).failed || []
+      failed_messages.any? do |failure|
         logger.error do
           "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
         end
@@ -43,7 +44,8 @@ module Shoryuken
     end
 
     def receive_messages(options)
-      client.receive_message(options.merge(queue_url: url)).messages.map { |m| Message.new(client, self, m) }
+      messages = client.receive_message(options.merge(queue_url: url)).messages || []
+      messages.map { |m| Message.new(client, self, m) }
     end
 
     def fifo?

data/lib/shoryuken/runner.rb

@@ -30,9 +30,6 @@ module Shoryuken
 
       loader = EnvironmentLoader.setup_options(options)
 
-      # When cli args exist, override options in config file
-      Shoryuken.options.merge!(options)
-
       daemonize(Shoryuken.options)
       write_pid(Shoryuken.options)
 

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '5.3.0'.freeze
+  VERSION = '6.2.1'.freeze
 end

data/lib/shoryuken.rb

@@ -23,6 +23,7 @@ require 'shoryuken/worker/default_executor'
 require 'shoryuken/worker/inline_executor'
 require 'shoryuken/worker_registry'
 require 'shoryuken/default_worker_registry'
+require 'shoryuken/default_exception_handler'
 require 'shoryuken/middleware/chain'
 require 'shoryuken/middleware/server/auto_delete'
 Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
@@ -73,6 +74,8 @@ module Shoryuken
     :sqs_client=,
     :sqs_client_receive_message_opts,
     :sqs_client_receive_message_opts=,
+    :exception_handlers,
+    :exception_handlers=,
     :options,
     :logger,
     :register_worker,
@@ -88,6 +91,10 @@ module Shoryuken
     :on,
     :cache_visibility_timeout?,
     :cache_visibility_timeout=,
+    :reloader,
+    :reloader=,
+    :enable_reloading,
+    :enable_reloading=,
     :delay
   )
 end

data/shoryuken.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Pablo Cantero']
   spec.email         = ['pablo@pablocantero.com']
   spec.description = spec.summary = 'Shoryuken is a super efficient AWS SQS thread based message processor'
-  spec.homepage      = 'https://github.com/phstc/shoryuken'
+  spec.homepage      = 'https://github.com/ruby-shoryuken/shoryuken'
   spec.license       = 'LGPL-3.0'
 
   spec.files         = `git ls-files -z`.split("\x0")

data/spec/shared_examples_for_active_job.rb

@@ -42,9 +42,11 @@ RSpec.shared_examples 'active_job_adapters' do
     context 'when fifo' do
       let(:fifo) { true }
 
-      it 'does not include job_id in the deduplication_id' do
+      it 'does not include job_id and enqueued_at in the deduplication_id' do
         expect(queue).to receive(:send_message) do |hash|
-          message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
+          message_deduplication_id = Digest::SHA256.hexdigest(
+            JSON.dump(job.serialize.except('job_id', 'enqueued_at'))
+          )
 
           expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
         end

data/spec/shoryuken/default_exception_handler_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+# rubocop:disable Metrics/BlockLength
+RSpec.describe Shoryuken::DefaultExceptionHandler do
+  class CustomErrorHandler
+    extend Shoryuken::Util
+
+    def self.call(_ex, queue, _msg)
+      logger.error("#{queue.to_s} failed to process the message")
+    end
+  end
+
+  before do
+    Shoryuken.worker_executor = Shoryuken::Worker::InlineExecutor
+    allow(manager).to receive(:async).and_return(manager)
+    allow(manager).to receive(:real_thread)
+    allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
+  end
+
+  after do
+    Shoryuken.worker_executor = Shoryuken::Worker::DefaultExecutor
+  end
+
+  let(:manager) { double Shoryuken::Manager }
+  let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 30 }
+  let(:queue) { 'default' }
+
+  let(:sqs_msg) do
+    double(
+      Shoryuken::Message,
+      queue_url: queue,
+      body: 'test',
+      message_attributes: {},
+      message_id: SecureRandom.uuid,
+      receipt_handle: SecureRandom.uuid
+    )
+  end
+
+  subject { Shoryuken::Processor.new(queue, sqs_msg) }
+
+  context "with default handler" do
+    before do
+      Shoryuken.exception_handlers = described_class
+    end
+
+    it "logs an error message" do
+      expect(Shoryuken::Logging.logger).to receive(:error).twice
+
+      allow_any_instance_of(TestWorker).to receive(:perform).and_raise(StandardError, "error")
+      allow(sqs_msg).to receive(:body)
+
+      expect { subject.process }.to raise_error(StandardError)
+    end
+  end
+
+  context "with custom handler" do
+    before do
+      Shoryuken.exception_handlers = [described_class, CustomErrorHandler]
+    end
+
+    it "logs default and custom error messages" do
+      expect(Shoryuken::Logging.logger).to receive(:error).twice
+      expect(Shoryuken::Logging.logger).to receive(:error).with("default failed to process the message").once
+
+      allow_any_instance_of(TestWorker).to receive(:perform).and_raise(StandardError, "error")
+      allow(sqs_msg).to receive(:body)
+
+      expect { subject.process }.to raise_error(StandardError)
+    end
+  end
+end