shoryuken 2.1.1 → 3.0.0

data/lib/shoryuken/environment_loader.rb

@@ -28,10 +28,8 @@ module Shoryuken
       prefix_active_job_queue_names
       parse_queues
       require_workers
-      initialize_aws
       validate_queues
       validate_workers
-      patch_deprecated_workers
     end
 
     private
@@ -49,19 +47,9 @@ module Shoryuken
       YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
     end
 
-    # DEPRECATED: Please use configure_server and configure_client in
-    # https://github.com/phstc/shoryuken/blob/a81637d577b36c5cf245882733ea91a335b6602f/lib/shoryuken.rb#L82
-    # Please delete this method afert next release (v2.0.12 or later)
-    def initialize_aws
-      unless Shoryuken.options[:aws].to_h.empty?
-        Shoryuken.logger.warn { '[DEPRECATION] aws in shoryuken.yml is deprecated. Please use configure_server and configure_client in your initializer' }
-      end
-      Shoryuken::AwsConfig.setup(Shoryuken.options[:aws])
-    end
-
     def initialize_logger
-      Shoryuken::Logging.initialize_logger(options[:logfile]) if options[:logfile]
-      Shoryuken.logger.level = Logger::DEBUG if options[:verbose]
+      Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
+      Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
     def load_rails
@@ -83,12 +71,11 @@ module Shoryuken
     end
 
     def merge_cli_defined_queues
-      cli_defined_queues = options.delete(:queues) || []
+      cli_defined_queues = options[:queues].to_a
 
       cli_defined_queues.each do |cli_defined_queue|
-        Shoryuken.options[:queues].delete_if do |config_file_queue|
-          config_file_queue[0] == cli_defined_queue[0]
-        end
+        # CLI defined queues override config_file defined queues
+        Shoryuken.options[:queues].delete_if { |config_file_queue| config_file_queue[0] == cli_defined_queue[0] }
 
         Shoryuken.options[:queues] << cli_defined_queue
       end
@@ -110,7 +97,7 @@ module Shoryuken
     end
 
     def parse_queue(queue, weight = nil)
-      [weight.to_i, 1].max.times { Shoryuken.queues << queue }
+      Shoryuken.add_queue(queue, [weight.to_i, 1].max)
     end
 
     def parse_queues
@@ -119,24 +106,6 @@ module Shoryuken
       end
     end
 
-    def patch_deprecated_workers
-      Shoryuken.worker_registry.queues.each do |queue|
-        Shoryuken.worker_registry.workers(queue).each do |worker_class|
-          if worker_class.instance_method(:perform).arity == 1
-            Shoryuken.logger.warn { "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)" }
-
-            worker_class.class_eval do
-              alias_method :deprecated_perform, :perform
-
-              def perform(sqs_msg, body = nil)
-                deprecated_perform(sqs_msg)
-              end
-            end
-          end
-        end
-      end
-    end
-
     def require_workers
       required = Shoryuken.options[:require]
 
@@ -156,7 +125,7 @@ module Shoryuken
 
       Shoryuken.queues.uniq.each do |queue|
         begin
-          Shoryuken::Client.queues queue
+          Shoryuken::Client.queues(queue)
         rescue Aws::SQS::Errors::NonExistentQueue
           non_existent_queues << queue
         end
@@ -166,13 +135,13 @@ module Shoryuken
     end
 
     def validate_workers
+      return if defined?(::ActiveJob)
+
       all_queues = Shoryuken.queues
       queues_with_workers = Shoryuken.worker_registry.queues
 
-      unless defined?(::ActiveJob)
-        (all_queues - queues_with_workers).each do |queue|
-          Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
-        end
+      (all_queues - queues_with_workers).each do |queue|
+        Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
       end
     end
   end

data/lib/shoryuken/fetcher.rb

@@ -1,73 +1,42 @@
 module Shoryuken
   class Fetcher
-    include Celluloid
     include Util
 
     FETCH_LIMIT = 10
 
-    def initialize(manager)
-      @manager = manager
+    def fetch(queue, available_processors)
+      started_at = Time.now
+
+      logger.debug { "Looking for new messages in '#{queue}'" }
+
+      begin
+        limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
+
+        sqs_msgs = Array(receive_messages(queue, limit))
+        logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
+        sqs_msgs
+      rescue => ex
+        logger.error { "Error fetching message: #{ex}" }
+        logger.error { ex.backtrace.first }
+        []
+      end
     end
 
+    private
+
     def receive_messages(queue, limit)
       # AWS limits the batch size by 10
       limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
-      options = (Shoryuken::AwsConfig.options[:receive_message] || {}).dup
+      options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
       options[:max_number_of_messages] = limit
       options[:message_attribute_names] = %w(All)
       options[:attribute_names] = %w(All)
 
-      Shoryuken::Client.queues(queue).receive_messages options
-    end
-
-    def fetch(queue, available_processors)
-      watchdog('Fetcher#fetch died') do
-        started_at = Time.now
-
-        logger.debug { "Looking for new messages in '#{queue}'" }
-
-        begin
-          batch = Shoryuken.worker_registry.batch_receive_messages?(queue)
-          limit = batch ? FETCH_LIMIT : available_processors
-
-          if (sqs_msgs = Array(receive_messages(queue, limit))).any?
-            logger.debug { "Found #{sqs_msgs.size} messages for '#{queue}'" }
-
-            if batch
-              @manager.async.assign(queue, patch_sqs_msgs!(sqs_msgs))
-            else
-              sqs_msgs.each { |sqs_msg| @manager.async.assign(queue, sqs_msg) }
-            end
-
-            @manager.async.rebalance_queue_weight!(queue)
-          else
-            logger.debug { "No message found for '#{queue}'" }
-
-            @manager.async.pause_queue!(queue)
-          end
-
-          logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
-        rescue => ex
-          logger.error { "Error fetching message: #{ex}" }
-          logger.error { ex.backtrace.first }
-        end
-
-        @manager.async.dispatch
-      end
-
-    end
-
-    private
-
-    def patch_sqs_msgs!(sqs_msgs)
-      sqs_msgs.instance_eval do
-        def message_id
-          "batch-with-#{size}-messages"
-        end
-      end
+      options.merge!(queue.options)
 
-      sqs_msgs
+      Shoryuken::Client.queues(queue.name).receive_messages(options)
     end
   end
 end

data/lib/shoryuken/launcher.rb

@@ -1,43 +1,19 @@
 module Shoryuken
   class Launcher
-    include Celluloid
     include Util
 
-    trap_exit :actor_died
-
-    attr_accessor :manager
-
     def initialize
-      @condvar = Celluloid::Condition.new
-      @manager = Shoryuken::Manager.new_link(@condvar)
-      @fetcher = Shoryuken::Fetcher.new_link(manager)
-
-      @done = false
-
-      manager.fetcher = @fetcher
+      @manager = Shoryuken::Manager.new(Shoryuken::Fetcher.new,
+                                        Shoryuken.options[:polling_strategy].new(Shoryuken.queues))
     end
 
     def stop(options = {})
-      watchdog('Launcher#stop') do
-        @done = true
-        @fetcher.terminate if @fetcher.alive?
-
-        manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken.options[:timeout])
-        @condvar.wait
-        manager.terminate
-      end
+      @manager.stop(shutdown: !options[:shutdown].nil?,
+                    timeout: Shoryuken.options[:timeout])
     end
 
     def run
-      watchdog('Launcher#run') do
-        manager.async.start
-      end
-    end
-
-    def actor_died(actor, reason)
-      return if @done
-      logger.warn { "Shoryuken died due to the following error, cannot recover, process exiting: #{reason}" }
-      exit 1
+      @manager.start
     end
   end
 end

data/lib/shoryuken/manager.rb

@@ -1,251 +1,133 @@
-require 'shoryuken/processor'
-require 'shoryuken/fetcher'
-
 module Shoryuken
   class Manager
-    include Celluloid
     include Util
 
-    attr_accessor :fetcher
+    BATCH_LIMIT        = 10
+    HEARTBEAT_INTERVAL = 0.1
 
-    trap_exit :processor_died
+    def initialize(fetcher, polling_strategy)
+      @count = Shoryuken.options.fetch(:concurrency, 25)
 
-    def initialize(condvar)
-      @count = Shoryuken.options[:concurrency] || 25
       raise(ArgumentError, "Concurrency value #{@count} is invalid, it needs to be a positive number") unless @count > 0
+
       @queues = Shoryuken.queues.dup.uniq
-      @finished = condvar
 
-      @done = false
+      @done = Concurrent::AtomicBoolean.new(false)
+      @dispatching = Concurrent::AtomicBoolean.new(false)
+
+      @fetcher = fetcher
+      @polling_strategy = polling_strategy
 
-      @busy  = []
-      @ready = @count.times.map { build_processor }
-      @threads = {}
+      @heartbeat = Concurrent::TimerTask.new(run_now: true,
+                                             execution_interval: HEARTBEAT_INTERVAL,
+                                             timeout_interval: 60) { dispatch }
+
+      @pool = Concurrent::FixedThreadPool.new(@count, max_queue: @count)
     end
 
     def start
       logger.info { 'Starting' }
 
-      dispatch
+      @heartbeat.execute
     end
 
     def stop(options = {})
-      watchdog('Manager#stop died') do
-        @done = true
-
-        if (callback = Shoryuken.stop_callback)
-          logger.info { 'Calling Shoryuken.on_stop block' }
-          callback.call
-        end
-
-        fire_event(:shutdown, true)
-
-        @fetcher.terminate if @fetcher.alive?
-
-        logger.info { "Shutting down #{@ready.size} quiet workers" }
+      @done.make_true
 
-        @ready.each do |processor|
-          processor.terminate if processor.alive?
-        end
-        @ready.clear
-
-        return after(0) { @finished.signal } if @busy.empty?
-
-        if options[:shutdown]
-          hard_shutdown_in(options[:timeout])
-        else
-          soft_shutdown(options[:timeout])
-        end
+      if (callback = Shoryuken.stop_callback)
+        logger.info { 'Calling Shoryuken.on_stop block' }
+        callback.call
       end
-    end
 
-    def processor_done(queue, processor)
-      watchdog('Manager#processor_done died') do
-        logger.debug { "Process done for '#{queue}'" }
+      fire_event(:shutdown, true)
 
-        @threads.delete(processor.object_id)
-        @busy.delete processor
+      logger.info { 'Shutting down workers' }
 
-        if stopped?
-          processor.terminate if processor.alive?
-          return after(0) { @finished.signal } if @busy.empty?
-        else
-          @ready << processor
-        end
-      end
-    end
+      @heartbeat.kill
 
-    def processor_died(processor, reason)
-      watchdog("Manager#processor_died died") do
-        logger.error { "Process died, reason: #{reason}" }
-
-        @threads.delete(processor.object_id)
-        @busy.delete processor
-
-        if stopped?
-          return after(0) { @finished.signal } if @busy.empty?
-        else
-          @ready << build_processor
-        end
-      end
-    end
-
-    def stopped?
-      @done
-    end
-
-    def assign(queue, sqs_msg)
-      watchdog('Manager#assign died') do
-        logger.debug { "Assigning #{sqs_msg.message_id}" }
-
-        processor = @ready.pop
-        @busy << processor
-
-        processor.async.process(queue, sqs_msg)
-      end
-    end
-
-    def rebalance_queue_weight!(queue)
-      watchdog('Manager#rebalance_queue_weight! died') do
-        if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
-          logger.info { "Increasing '#{queue}' weight to #{current + 1}, max: #{original}" }
-
-          @queues << queue
-        end
+      if options[:shutdown]
+        hard_shutdown_in(options[:timeout])
+      else
+        soft_shutdown
       end
     end
 
-    def pause_queue!(queue)
-      return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
-
-      logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty" }
-
-      @queues.delete(queue)
-
-      after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
+    def processor_done(queue)
+      logger.debug { "Process done for '#{queue}'" }
     end
 
+    private
 
     def dispatch
-      return if stopped?
+      return if @done.true?
+      return unless @dispatching.make_true
 
-      logger.debug { "Ready: #{@ready.size}, Busy: #{@busy.size}, Active Queues: #{unparse_queues(@queues)}" }
+      return if ready.zero?
+      return unless (queue = @polling_strategy.next_queue)
 
-      if @ready.empty?
-        logger.debug { 'Pausing fetcher, because all processors are busy' }
-
-        dispatch_later
-        return
-      end
-
-      if (queue = next_queue)
-        @fetcher.async.fetch(queue, @ready.size)
-      else
-        logger.debug { 'Pausing fetcher, because all queues are paused' }
+      logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
-        @fetcher_paused = true
-      end
-    end
-
-    def real_thread(proxy_id, thr)
-      @threads[proxy_id] = thr
+      batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
+    ensure
+      @dispatching.make_false
     end
 
-    private
-
-    def dispatch_later
-      @_dispatch_timer ||= after(1) do
-        @_dispatch_timer = nil
-        dispatch
-      end
+    def busy
+      @count - ready
     end
 
-    def build_processor
-      processor = Processor.new_link(current_actor)
-      processor.proxy_id = processor.object_id
-      processor
+    def ready
+      @pool.remaining_capacity
     end
 
-    def restart_queue!(queue)
-      return if stopped?
-
-      unless @queues.include? queue
-        logger.debug { "Restarting '#{queue}'" }
-
-        @queues << queue
-
-        if @fetcher_paused
-          logger.debug { 'Restarting fetcher' }
-
-          @fetcher_paused = false
+    def assign(queue, sqs_msg)
+      logger.debug { "Assigning #{sqs_msg.message_id}" }
 
-          dispatch
-        end
-      end
+      @pool.post { Processor.new(self).process(queue, sqs_msg) }
     end
 
-    def current_queue_weight(queue)
-      queue_weight(@queues, queue)
+    def dispatch_batch(queue)
+      batch = @fetcher.fetch(queue, BATCH_LIMIT)
+      @polling_strategy.messages_found(queue.name, batch.size)
+      assign(queue.name, patch_batch!(batch))
     end
 
-    def original_queue_weight(queue)
-      queue_weight(Shoryuken.queues, queue)
+    def dispatch_single_messages(queue)
+      messages = @fetcher.fetch(queue, ready)
+      @polling_strategy.messages_found(queue.name, messages.size)
+      messages.each { |message| assign(queue.name, message) }
     end
 
-    def queue_weight(queues, queue)
-      queues.count { |q| q == queue }
+    def batched_queue?(queue)
+      Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
 
-    def next_queue
-      return nil if @queues.empty?
-
-      # get/remove the first queue in the list
-      queue = @queues.shift
-
-      unless defined?(::ActiveJob) || !Shoryuken.worker_registry.workers(queue).empty?
-        # when no worker registered pause the queue to avoid endless recursion
-        logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because no workers registered" }
-
-        after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
+    def soft_shutdown
+      @pool.shutdown
+      @pool.wait_for_termination
+    end
 
-        return next_queue
+    def hard_shutdown_in(delay)
+      if busy > 0
+        logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
       end
 
-      # add queue back to the end of the list
-      @queues << queue
-
-      queue
-    end
+      @pool.shutdown
 
-    def soft_shutdown(delay)
-      logger.info { "Waiting for #{@busy.size} busy workers" }
+      return if @pool.wait_for_termination(delay)
 
-      if @busy.size > 0
-        after(delay) { soft_shutdown(delay) }
-      else
-        @finished.signal
-      end
+      logger.info { "Hard shutting down #{busy} busy workers" }
+      @pool.kill
     end
 
-    def hard_shutdown_in(delay)
-      logger.info { "Waiting for #{@busy.size} busy workers" }
-      logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
-
-      after(delay) do
-        watchdog('Manager#hard_shutdown_in died') do
-          if @busy.size > 0
-            logger.info { "Hard shutting down #{@busy.size} busy workers" }
-
-            @busy.each do |processor|
-              if processor.alive? && t = @threads.delete(processor.object_id)
-                t.raise Shutdown
-              end
-            end
-          end
-
-          @finished.signal
+    def patch_batch!(sqs_msgs)
+      sqs_msgs.instance_eval do
+        def message_id
+          "batch-with-#{size}-messages"
         end
       end
+
+      sqs_msgs
     end
   end
 end

data/lib/shoryuken/message.rb

@@ -3,19 +3,10 @@ module Shoryuken
     attr_accessor :client, :queue_url, :queue_name, :data
 
     def initialize(client, queue, data)
-      self.client = client
-      self.data = data
-
-      if queue.is_a?(Shoryuken::Queue)
-        self.queue_url = queue.url
-        self.queue_name = queue.name
-      else
-        # TODO: Remove next major release
-        Shoryuken.logger.warn do
-          '[DEPRECATION] Passing a queue url into Shoryuken::Message is deprecated, please pass the queue itself'
-        end
-        self.queue_url = queue
-      end
+      self.client     = client
+      self.data       = data
+      self.queue_url  = queue.url
+      self.queue_name = queue.name
     end
 
     def delete

data/lib/shoryuken/middleware/chain.rb

@@ -102,32 +102,15 @@ module Shoryuken
 
     class Entry
       attr_reader :klass
+
       def initialize(klass, *args)
         @klass = klass
         @args  = args
-
-        patch_deprecated_middleware!(klass)
       end
 
       def make_new
         @klass.new(*@args)
       end
-
-      private
-
-      def patch_deprecated_middleware!(klass)
-        if klass.instance_method(:call).arity == 3
-          Shoryuken.logger.warn { "[DEPRECATION] #{klass.name}#call(worker_instance, queue, sqs_msg) is deprecated. Please use #{klass.name}#call(worker_instance, queue, sqs_msg, body)" }
-
-          klass.class_eval do
-            alias_method :deprecated_call, :call
-
-            def call(worker_instance, queue, sqs_msg, body = nil, &block)
-              deprecated_call(worker_instance, queue, sqs_msg, &block)
-            end
-          end
-        end
-      end
     end
   end
 end

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -1,33 +1,32 @@
-require 'celluloid/current' unless defined?(Celluloid)
-
 module Shoryuken
   module Middleware
     module Server
       class AutoExtendVisibility
+        include Util
+
         EXTEND_UPFRONT_SECONDS = 5
 
         def call(worker, queue, sqs_msg, body)
-          timer = auto_visibility_timer(worker, queue, sqs_msg, body)
-          begin
-            yield
-          ensure
-            if timer
-              timer.cancel
-              @visibility_extender.terminate
-            end
+          if sqs_msg.is_a?(Array)
+            logger.warn { "Auto extend visibility isn't supported for batch workers" }
+            return yield
           end
+
+          timer = auto_visibility_timer(worker, queue, sqs_msg, body)
+          yield
+        ensure
+          timer.kill if timer
         end
 
         private
 
         class MessageVisibilityExtender
-          include Celluloid
           include Util
 
           def auto_extend(worker, queue, sqs_msg, body)
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
-            every(queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
+            Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
               begin
                 logger.debug do
                   "Extending message #{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
@@ -48,8 +47,8 @@ module Shoryuken
 
         def auto_visibility_timer(worker, queue, sqs_msg, body)
           return unless worker.class.auto_visibility_timeout?
-          @visibility_extender = MessageVisibilityExtender.new_link
-          @visibility_extender.auto_extend(worker, queue, sqs_msg, body)
+
+          MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body).tap(&:execute)
         end
       end
     end