shoryuken 4.0.1 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1f28614380213b13c17dea0f85aae332f8e335f6
-  data.tar.gz: 34e084982c8aa8ec9b0306f04f548cc1321eec13
+SHA256:
+  metadata.gz: ce36f4dac1c75e383185df15db196aec57c968dda1a5823b8d944ad4a9675906
+  data.tar.gz: a460a585e97a6c0a3432f6b28bec254d494900672a06bffbd7461e830a8de0cf
 SHA512:
-  metadata.gz: 40a160284600447798e3e1ebc61ec1e8bf596b0aa9d5558301d7f647e1ab1af285b845a9ec052eb52e157714d4066d3888f6f2192762898d9fc52decf1503a3b
-  data.tar.gz: 7f3d7584167804ad724dc3ca3a661cef8a4a60cadccebe333ceacb10333c0eb1bc7db9247439ddce875a1b7cb771769e9516afcda0ae5703991db780acd165d3
+  metadata.gz: 833bab295345b25456f617f36e4ec2f2cfd0d823a01ddb87d8f528597ab79b91ce0ae8c67b47ffc2a9d1400f0a5cf62df3125fbdfbb49abd3a5e4a5eb3b7b0ee
+  data.tar.gz: 91a4355cd9e180f006e8768bde9627c1df9cf0057d294bdfcb422f479de2ef1784aa5b904c7b97dac8ed7bb7b9879bdea56eb4cd00d51ff456d3b4555f88c991

data/.gitignore

@@ -25,3 +25,4 @@ shoryuken.yml
 *.log
 .env
 rubocop.html
+.byebug_history

data/.rubocop.yml

@@ -1,7 +1,7 @@
 AllCops:
   Exclude:
     - '**/Gemfile'
-  TargetRubyVersion: 2.1
+  TargetRubyVersion: 2.3
 
 Metrics/PerceivedComplexity:
   Enabled: false

data/.travis.yml

@@ -3,10 +3,11 @@ language: ruby
 rvm:
   - 2.0.0
   - 2.1.10
-  - 2.2.9
-  - 2.3.6
-  - 2.4.3
-  - 2.5.0
+  - 2.2.10
+  - 2.3.8
+  - 2.4.4
+  - 2.5.1
+  - 2.6.3
 
 notifications:
   email:
@@ -23,7 +24,7 @@ env:
 script: bundle exec rspec spec
 
 before_install:
-  - gem update bundler
+  - gem install bundler -v '< 2'
 
 after_success:
   - bundle exec codeclimate-test-reporter

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+## [v5.0.0] - 2019-06-18
+
+- Fix bug where empty queues were not paused in batch processing mode
+  - [#569](https://github.com/phstc/shoryuken/pull/569)
+
+- Preserve batch limit when receiving messages from a FIFO queue
+  - [#563](https://github.com/phstc/shoryuken/pull/563)
+
+- Replace static options with instance options
+  - [#534](https://github.com/phstc/shoryuken/pull/534)
+
+## [v4.0.3] - 2019-01-06
+
+- Support delay per processing group
+  - [#543](https://github.com/phstc/shoryuken/pull/543)
+
+## [v4.0.2] - 2018-11-26
+
+- Fix the delegated methods to public warning
+  - [#536](https://github.com/phstc/shoryuken/pull/536)
+
+- Specify exception class to `raise_error` matcher warning
+  - [#537](https://github.com/phstc/shoryuken/pull/537)
+
+- Fix spelling of "visibility"
+  - [#538](https://github.com/phstc/shoryuken/pull/538)
+
 ## [v4.0.1] - 2018-11-21
 
 - Allow caching visibility_timeout lookups

data/lib/shoryuken/environment_loader.rb

@@ -99,7 +99,7 @@ module Shoryuken
 
     def prefix_active_job_queue_names
       return unless Shoryuken.active_job?
-      return unless Shoryuken.active_job_queue_name_prefixing
+      return unless Shoryuken.active_job_queue_name_prefixing?
 
       Shoryuken.options[:queues].to_a.map! do |queue_name, weight|
         prefix_active_job_queue_name(queue_name, weight)
@@ -122,7 +122,7 @@ module Shoryuken
 
     def parse_queues
       if Shoryuken.options[:queues].to_a.any?
-        Shoryuken.add_group('default', Shoryuken.options.fetch(:concurrency, 25))
+        Shoryuken.add_group('default', Shoryuken.options[:concurrency])
 
         Shoryuken.options[:queues].to_a.each do |queue, weight|
           parse_queue(queue, weight, 'default')
@@ -130,7 +130,8 @@ module Shoryuken
       end
 
       Shoryuken.options[:groups].to_a.each do |group, options|
-        Shoryuken.add_group(group, options.fetch(:concurrency, 25))
+        Shoryuken.add_group(group, options[:concurrency], delay: options[:delay])
+
         options[:queues].to_a.each do |queue, weight|
           parse_queue(queue, weight, group)
         end

data/lib/shoryuken/fetcher.rb

@@ -14,7 +14,7 @@ module Shoryuken
 
         logger.debug { "Looking for new messages in #{queue}" }
 
-        sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
+        sqs_msgs = Array(receive_messages(queue, limit))
 
         logger.debug { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
         logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
@@ -49,16 +49,7 @@ module Shoryuken
 
       shoryuken_queue = Shoryuken::Client.queues(queue.name)
 
-      # For FIFO queues we want to make sure we process one message per group at the time
-      # if we set max_number_of_messages greater than 1,
-      # SQS may return more than one message for the same message group
-      # since Shoryuken uses threads, it will try to process more than one at once
-      # > The message group ID is the tag that specifies that a message belongs to a specific message group.
-      # > Messages that belong to the same message group are always processed one by one,
-      # > in a strict order relative to the message group
-      # > (however, messages that belong to different message groups might be processed out of order).
-      # > https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html
-      options[:max_number_of_messages]  = shoryuken_queue.fifo? ? 1 : max_number_of_messages(limit, options)
+      options[:max_number_of_messages]  = max_number_of_messages(shoryuken_queue, limit, options)
       options[:message_attribute_names] = %w[All]
       options[:attribute_names]         = %w[All]
 
@@ -67,7 +58,18 @@ module Shoryuken
       shoryuken_queue.receive_messages(options)
     end
 
-    def max_number_of_messages(limit, options)
+    def max_number_of_messages(shoryuken_queue, limit, options)
+      # For FIFO queues we want to make sure we process one message per group at a time
+      # if we set max_number_of_messages greater than 1,
+      # SQS may return more than one message for the same message group
+      # since Shoryuken uses threads, it will try to process more than one message at once
+      # > The message group ID is the tag that specifies that a message belongs to a specific message group.
+      # > Messages that belong to the same message group are always processed one by one,
+      # > in a strict order relative to the message group
+      # > (however, messages that belong to different message groups might be processed out of order).
+      # > https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html
+      limit = 1 if shoryuken_queue.fifo? && !batched_queue?(shoryuken_queue)
+
       [limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
     end
 
@@ -77,5 +79,9 @@ module Shoryuken
 
       options.to_h.dup
     end
+
+    def batched_queue?(queue)
+      Shoryuken.worker_registry.batch_receive_messages?(queue.name)
+    end
   end
-end
+end

data/lib/shoryuken/launcher.rb

@@ -72,7 +72,7 @@ module Shoryuken
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(
           Shoryuken::Fetcher.new(group),
-          Shoryuken.polling_strategy(group).new(options[:queues]),
+          Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
           options[:concurrency],
           executor
         )

data/lib/shoryuken/manager.rb

@@ -43,8 +43,8 @@ module Shoryuken
       logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
       batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
-    rescue => ex
-      handle_dispatch_error(ex)
+    rescue => e
+      handle_dispatch_error(e)
     ensure
       dispatch_loop
     end
@@ -74,9 +74,9 @@ module Shoryuken
     end
 
     def dispatch_batch(queue)
-      return if (batch = @fetcher.fetch(queue, BATCH_LIMIT)).none?
+      batch = @fetcher.fetch(queue, BATCH_LIMIT)
       @polling_strategy.messages_found(queue.name, batch.size)
-      assign(queue.name, patch_batch!(batch))
+      assign(queue.name, patch_batch!(batch)) if batch.any?
     end
 
     def dispatch_single_messages(queue)

data/lib/shoryuken/options.rb

@@ -4,7 +4,7 @@ module Shoryuken
       concurrency: 25,
       queues: [],
       aws: {},
-      delay: 0,
+      delay: 0.0,
       timeout: 8,
       lifecycle_events: {
         startup: [],
@@ -14,221 +14,166 @@ module Shoryuken
       }
     }.freeze
 
-    @@groups                          = {}
-    @@worker_registry                 = DefaultWorkerRegistry.new
-    @@active_job_queue_name_prefixing = false
-    @@sqs_client                      = nil
-    @@sqs_client_receive_message_opts = {}
-    @@start_callback                  = nil
-    @@stop_callback                   = nil
-    @@worker_executor                 = Worker::DefaultExecutor
-    @@launcher_executor               = nil
-    @@cache_visiblity_timeout         = false
-
-    class << self
-      def active_job?
-        defined?(::ActiveJob)
-      end
-
-      def add_group(group, concurrency)
-        groups[group] ||= {
-          concurrency: concurrency,
-          queues: []
-        }
-      end
-
-      def groups
-        @@groups
-      end
-
-      def add_queue(queue, weight, group)
-        weight.times do
-          groups[group][:queues] << queue
-        end
-      end
-
-      def ungrouped_queues
-        groups.values.flat_map { |options| options[:queues] }
-      end
-
-      def worker_registry
-        @@worker_registry
-      end
-
-      def worker_registry=(worker_registry)
-        @@worker_registry = worker_registry
-      end
-
-      def worker_executor
-        @@worker_executor
-      end
-
-      def worker_executor=(worker_executor)
-        @@worker_executor = worker_executor
-      end
-
-      def launcher_executor
-        @@launcher_executor
-      end
-
-      def launcher_executor=(launcher_executor)
-        @@launcher_executor = launcher_executor
-      end
+    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :default_worker_options, :groups,
+                  :launcher_executor, :sqs_client, :sqs_client_receive_message_opts,
+                  :start_callback, :worker_executor, :worker_registry
+
+    def initialize
+      self.groups = {}
+      self.worker_registry = DefaultWorkerRegistry.new
+      self.active_job_queue_name_prefixing = false
+      self.worker_executor = Worker::DefaultExecutor
+      self.cache_visibility_timeout = false
+      # this is needed for keeping backward compatibility
+      @sqs_client_receive_message_opts ||= {}
+    end
 
-      def polling_strategy(group)
-        strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
-
-        case strategy
-        when 'WeightedRoundRobin', nil # Default case
-          Polling::WeightedRoundRobin
-        when 'StrictPriority'
-          Polling::StrictPriority
-        when Class
-          strategy
-        else
-          raise ArgumentError, "#{strategy} is not a valid polling_strategy"
-        end
-      end
+    def active_job?
+      defined?(::ActiveJob)
+    end
 
-      def start_callback
-        @@start_callback
-      end
+    def add_group(group, concurrency = nil, delay: nil)
+      concurrency ||= options[:concurrency]
+      delay ||= options[:delay]
 
-      def start_callback=(start_callback)
-        @@start_callback = start_callback
-      end
+      groups[group] ||= {
+        concurrency: concurrency,
+        delay: delay,
+        queues: []
+      }
+    end
 
-      def stop_callback
-        @@stop_callback
+    def add_queue(queue, weight, group)
+      weight.times do
+        groups[group][:queues] << queue
       end
+    end
 
-      def stop_callback=(stop_callback)
-        @@stop_callback = stop_callback
-      end
+    def ungrouped_queues
+      groups.values.flat_map { |options| options[:queues] }
+    end
 
-      def active_job_queue_name_prefixing
-        @@active_job_queue_name_prefixing
+    def polling_strategy(group)
+      strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
+      case strategy
+      when 'WeightedRoundRobin', nil # Default case
+        Polling::WeightedRoundRobin
+      when 'StrictPriority'
+        Polling::StrictPriority
+      when Class
+        strategy
+      else
+        raise ArgumentError, "#{strategy} is not a valid polling_strategy"
       end
+    end
 
-      def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
-        @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
-      end
+    def delay(group)
+      groups[group].to_h.fetch(:delay, options[:delay]).to_f
+    end
 
-      def sqs_client
-        @@sqs_client ||= Aws::SQS::Client.new
-      end
+    def sqs_client
+      @sqs_client ||= Aws::SQS::Client.new
+    end
 
-      def sqs_client=(sqs_client)
-        @@sqs_client = sqs_client
-      end
+    def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
+      @sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
+    end
 
-      def sqs_client_receive_message_opts
-        @@sqs_client_receive_message_opts
-      end
+    def options
+      @options ||= DEFAULTS.dup
+    end
 
-      def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
-        @@sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
-      end
+    def logger
+      Shoryuken::Logging.logger
+    end
 
-      def options
-        @@options ||= DEFAULTS.dup
-      end
+    def register_worker(*args)
+      worker_registry.register_worker(*args)
+    end
 
-      def logger
-        Shoryuken::Logging.logger
-      end
+    def configure_server
+      yield self if server?
+    end
 
-      def register_worker(*args)
-        @@worker_registry.register_worker(*args)
-      end
+    def server_middleware
+      @_server_chain ||= default_server_middleware
+      yield @_server_chain if block_given?
+      @_server_chain
+    end
 
-      def configure_server
-        yield self if server?
-      end
+    def configure_client
+      yield self unless server?
+    end
 
-      def server_middleware
-        @@server_chain ||= default_server_middleware
-        yield @@server_chain if block_given?
-        @@server_chain
-      end
+    def client_middleware
+      @_client_chain ||= default_client_middleware
+      yield @_client_chain if block_given?
+      @_client_chain
+    end
 
-      def configure_client
-        yield self unless server?
-      end
+    def default_worker_options
+      @default_worker_options ||= {
+        'queue' => 'default',
+        'delete' => false,
+        'auto_delete' => false,
+        'auto_visibility_timeout' => false,
+        'retry_intervals' => nil,
+        'batch' => false
+      }
+    end
 
-      def client_middleware
-        @@client_chain ||= default_client_middleware
-        yield @@client_chain if block_given?
-        @@client_chain
-      end
+    def on_start(&block)
+      self.start_callback = block
+    end
 
-      def default_worker_options
-        @@default_worker_options ||= {
-          'queue'                   => 'default',
-          'delete'                  => false,
-          'auto_delete'             => false,
-          'auto_visibility_timeout' => false,
-          'retry_intervals'         => nil,
-          'batch'                   => false
-        }
-      end
+    def on_stop(&block)
+      self.stop_callback = block
+    end
 
-      def default_worker_options=(default_worker_options)
-        @@default_worker_options = default_worker_options
-      end
+    # Register a block to run at a point in the Shoryuken lifecycle.
+    # :startup, :quiet or :shutdown are valid events.
+    #
+    #   Shoryuken.configure_server do |config|
+    #     config.on(:shutdown) do
+    #       puts "Goodbye cruel world!"
+    #     end
+    #   end
+    def on(event, &block)
+      fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+      fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+
+      options[:lifecycle_events][event] << block
+    end
 
-      def on_start(&block)
-        @@start_callback = block
-      end
+    def server?
+      defined?(Shoryuken::CLI)
+    end
 
-      def on_stop(&block)
-        @@stop_callback = block
-      end
+    def cache_visibility_timeout?
+      @cache_visibility_timeout
+    end
 
-      # Register a block to run at a point in the Shoryuken lifecycle.
-      # :startup, :quiet or :shutdown are valid events.
-      #
-      #   Shoryuken.configure_server do |config|
-      #     config.on(:shutdown) do
-      #       puts "Goodbye cruel world!"
-      #     end
-      #   end
-      def on(event, &block)
-        fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-        fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
-        options[:lifecycle_events][event] << block
-      end
+    def active_job_queue_name_prefixing?
+      @active_job_queue_name_prefixing
+    end
 
-      private
-
-      def default_server_middleware
-        Middleware::Chain.new do |m|
-          m.add Middleware::Server::Timing
-          m.add Middleware::Server::ExponentialBackoffRetry
-          m.add Middleware::Server::AutoDelete
-          m.add Middleware::Server::AutoExtendVisibility
-          if defined?(::ActiveRecord::Base)
-            require 'shoryuken/middleware/server/active_record'
-            m.add Middleware::Server::ActiveRecord
-          end
+    private
+
+    def default_server_middleware
+      Middleware::Chain.new do |m|
+        m.add Middleware::Server::Timing
+        m.add Middleware::Server::ExponentialBackoffRetry
+        m.add Middleware::Server::AutoDelete
+        m.add Middleware::Server::AutoExtendVisibility
+        if defined?(::ActiveRecord::Base)
+          require 'shoryuken/middleware/server/active_record'
+          m.add Middleware::Server::ActiveRecord
         end
       end
+    end
 
-      def default_client_middleware
-        Middleware::Chain.new
-      end
-
-      def server?
-        defined?(Shoryuken::CLI)
-      end
-
-      def cache_visiblity_timeout?
-        @@cache_visiblity_timeout
-      end
-
-      def cache_visiblity_timeout=(cache_visiblity_timeout)
-        @@cache_visiblity_timeout = cache_visiblity_timeout
-      end
+    def default_client_middleware
+      Middleware::Chain.new
     end
   end
 end

data/lib/shoryuken/polling/base.rb

@@ -57,10 +57,8 @@ module Shoryuken
         end
       end
 
-      private
-
       def delay
-        Shoryuken.options[:delay].to_f
+        @delay || Shoryuken.options[:delay].to_f
       end
     end
   end

data/lib/shoryuken/polling/strict_priority.rb

@@ -1,7 +1,7 @@
 module Shoryuken
   module Polling
     class StrictPriority < BaseStrategy
-      def initialize(queues)
+      def initialize(queues, delay = nil)
         # Priority ordering of the queues, highest priority first
         @queues = queues
                   .group_by { |q| q }
@@ -12,6 +12,7 @@ module Shoryuken
         @paused_until = queues
                         .each_with_object({}) { |queue, h| h[queue] = Time.at(0) }
 
+        @delay = delay
         # Start queues at 0
         reset_next_queue
       end

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -1,10 +1,11 @@
 module Shoryuken
   module Polling
     class WeightedRoundRobin < BaseStrategy
-      def initialize(queues)
+      def initialize(queues, delay = nil)
         @initial_queues = queues
         @queues = queues.dup.uniq
         @paused_queues = []
+        @delay = delay
       end
 
       def next_queue

data/lib/shoryuken/queue.rb

@@ -14,9 +14,9 @@ module Shoryuken
     end
 
     def visibility_timeout
-      # Always lookup for the latest visiblity when cache is disabled
+      # Always lookup for the latest visibility when cache is disabled
       # setting it to nil, forces re-lookup
-      @_visibility_timeout = nil unless Shoryuken.cache_visiblity_timeout?
+      @_visibility_timeout = nil unless Shoryuken.cache_visibility_timeout?
       @_visibility_timeout ||= queue_attributes.attributes[VISIBILITY_TIMEOUT_ATTR].to_i
     end
 
@@ -66,7 +66,7 @@ module Shoryuken
     end
 
     def set_name_and_url(name_or_url)
-      if name_or_url.start_with?('https://sqs.')
+      if name_or_url.include?('://')
         set_by_url(name_or_url)
 
         # anticipate the fifo? checker for validating the queue URL

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '4.0.1'.freeze
+  VERSION = '5.0.0'.freeze
 end

data/lib/shoryuken/worker.rb

@@ -17,9 +17,9 @@ module Shoryuken
       alias_method :perform_at, :perform_in
 
       def server_middleware
-        @server_chain ||= Shoryuken.server_middleware.dup
-        yield @server_chain if block_given?
-        @server_chain
+        @_server_chain ||= Shoryuken.server_middleware.dup
+        yield @_server_chain if block_given?
+        @_server_chain
       end
 
       def shoryuken_options(opts = {})

data/lib/shoryuken.rb

@@ -41,8 +41,12 @@ require 'shoryuken/options'
 module Shoryuken
   extend SingleForwardable
 
+  def self.shoryuken_options
+    @_shoryuken_options ||= Shoryuken::Options.new
+  end
+
   def_delegators(
-    :'Shoryuken::Options',
+    :shoryuken_options,
     :active_job?,
     :add_group,
     :groups,
@@ -59,7 +63,7 @@ module Shoryuken
     :start_callback=,
     :stop_callback,
     :stop_callback=,
-    :active_job_queue_name_prefixing,
+    :active_job_queue_name_prefixing?,
     :active_job_queue_name_prefixing=,
     :sqs_client,
     :sqs_client=,
@@ -78,8 +82,9 @@ module Shoryuken
     :on_start,
     :on_stop,
     :on,
-    :cache_visiblity_timeout?,
-    :cache_visiblity_timeout=
+    :cache_visibility_timeout?,
+    :cache_visibility_timeout=,
+    :delay
   )
 end
 

data/shoryuken.gemspec

@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'dotenv'
   spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'rake'

data/spec/shoryuken/environment_loader_spec.rb

@@ -4,7 +4,7 @@ require 'active_job'
 RSpec.describe Shoryuken::EnvironmentLoader do
   subject { described_class.new({}) }
 
-  describe '#parse_queues' do
+  describe '#parse_queues loads default queues' do
     before do
       allow(subject).to receive(:load_rails)
       allow(subject).to receive(:prefix_active_job_queue_names)
@@ -22,6 +22,29 @@ RSpec.describe Shoryuken::EnvironmentLoader do
     end
   end
 
+  describe '#parse_queues includes delay per groups' do
+    before do
+      allow(subject).to receive(:load_rails)
+      allow(subject).to receive(:prefix_active_job_queue_names)
+      allow(subject).to receive(:require_workers)
+      allow(subject).to receive(:validate_queues)
+      allow(subject).to receive(:validate_workers)
+      allow(subject).to receive(:patch_deprecated_workers)
+    end
+
+    specify do
+      Shoryuken.options[:queues] = ['queue1', 'queue2'] # default queues
+      Shoryuken.options[:groups] = [[ 'custom', { queues: ['queue3'], delay: 25 }]]
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to eq(%w[queue1 queue2])
+      expect(Shoryuken.groups['default'][:delay]).to eq(Shoryuken.options[:delay])
+      expect(Shoryuken.groups['custom'][:queues]).to eq(%w[queue3])
+      expect(Shoryuken.groups['custom'][:delay]).to eq(25)
+    end
+  end
+
+
   describe '#prefix_active_job_queue_names' do
     before do
       allow(subject).to receive(:load_rails)

data/spec/shoryuken/fetcher_spec.rb

@@ -103,9 +103,9 @@ RSpec.describe Shoryuken::Fetcher do
 
     context 'when FIFO' do
       let(:limit) { 10 }
-      let(:queue) { instance_double('Shoryuken::Queue', fifo?: true) }
+      let(:queue) { instance_double('Shoryuken::Queue', fifo?: true, name: queue_name) }
 
-      it 'polls one message at the time' do
+      it 'polls one message at a time' do
         # see https://github.com/phstc/shoryuken/pull/530
 
         allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
@@ -115,6 +115,21 @@ RSpec.describe Shoryuken::Fetcher do
 
         subject.fetch(queue_config, limit)
       end
+
+      context 'with batch=true' do
+        it 'polls the provided limit' do
+          # see https://github.com/phstc/shoryuken/pull/530
+
+          allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+          allow(Shoryuken.worker_registry).to receive(:batch_receive_messages?).with(queue.name).and_return(true)
+
+          expect(queue).to receive(:receive_messages).with(
+            max_number_of_messages: limit, attribute_names: ['All'], message_attribute_names: ['All']
+          ).and_return([])
+
+          subject.fetch(queue_config, limit)
+        end
+      end
     end
   end
 end

data/spec/shoryuken/manager_spec.rb

@@ -72,11 +72,27 @@ RSpec.describe Shoryuken::Manager do
       expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
       expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
       expect(Shoryuken::Processor).to receive(:process).with(q, message)
-      expect(Shoryuken.logger).to_not receive(:info)
+      expect(Shoryuken.logger).to receive(:info).never
 
       subject.send(:dispatch)
     end
 
+    context 'and there are no messages in the queue' do
+      specify do
+        messages = %w[]
+        q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
+
+        expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
+        expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
+        expect(polling_strategy).to receive(:messages_found).with(q.name, 0)
+        expect(Shoryuken.logger).to receive(:info).never
+        expect(Shoryuken::Processor).to receive(:process).never
+        expect_any_instance_of(described_class).to receive(:assign).never
+
+        subject.send(:dispatch)
+      end
+    end
+
     context 'when batch' do
       specify do
         messages = %w[test1 test2 test3]
@@ -86,10 +102,27 @@ RSpec.describe Shoryuken::Manager do
         expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
         allow(subject).to receive(:batched_queue?).with(q).and_return(true)
         expect(Shoryuken::Processor).to receive(:process).with(q, messages)
-        expect(Shoryuken.logger).to_not receive(:info)
+        expect(Shoryuken.logger).to receive(:info).never
 
         subject.send(:dispatch)
       end
+
+      context 'and there are no messages in the queue' do
+        specify do
+          messages = %w[]
+          q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
+
+          expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
+          expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
+          allow(subject).to receive(:batched_queue?).with(q).and_return(true)
+          expect(polling_strategy).to receive(:messages_found).with(q.name, 0)
+          expect(Shoryuken.logger).to receive(:info).never
+          expect(Shoryuken::Processor).to receive(:process).never
+          expect_any_instance_of(described_class).to receive(:assign).never
+
+          subject.send(:dispatch)
+        end
+      end
     end
   end
 

data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb

@@ -53,7 +53,7 @@ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
     allow(sqs_msg).to receive(:queue) { sqs_queue }
     expect(sqs_msg).to_not receive(:change_visibility)
 
-    expect { Runner.new.run_and_raise(TestWorker.new, queue, sqs_msg) }.to raise_error
+    expect { Runner.new.run_and_raise(TestWorker.new, queue, sqs_msg) }.to raise_error(RuntimeError)
   end
 
   it 'does not extend message visibility if auto_visibility_timeout is not true' do

data/spec/shoryuken/options_spec.rb

@@ -1,19 +1,37 @@
 require 'spec_helper'
 
 RSpec.describe Shoryuken::Options do
-  describe '.add_group' do
+  subject { Shoryuken.shoryuken_options }
+
+  describe '.add_group adds queues and optional delay' do
     before do
       Shoryuken.groups.clear
       Shoryuken.add_group('group1', 25)
       Shoryuken.add_group('group2', 25)
+      Shoryuken.add_group('group3', 25, delay: 5)
+    end
+
+    specify do
+      subject.add_queue('queue1', 1, 'group1')
+      subject.add_queue('queue2', 2, 'group2')
+      subject.add_queue('queue3', 1, 'group3')
+
+      expect(subject.groups['group1'][:queues]).to eq(%w[queue1])
+      expect(subject.groups['group2'][:queues]).to eq(%w[queue2 queue2])
+      expect(subject.groups['group3'][:queues]).to eq(%w[queue3])
+      expect(subject.groups['group3'][:delay]).to eq(5)
     end
+  end
 
+  describe '.delay works for each group' do
     specify do
-      described_class.add_queue('queue1', 1, 'group1')
-      described_class.add_queue('queue2', 2, 'group2')
+      Shoryuken.add_group('group1', 25)
+      Shoryuken.add_group('group2', 25, delay: 5)
+      subject.add_queue('queue1', 1, 'group1')
+      subject.add_queue('queue2', 2, 'group2')
 
-      expect(described_class.groups['group1'][:queues]).to eq(%w[queue1])
-      expect(described_class.groups['group2'][:queues]).to eq(%w[queue2 queue2])
+      expect(subject.delay('group1')).to eq(Shoryuken.options[:delay])
+      expect(subject.delay('group2')).to eq(5.0)
     end
   end
 
@@ -25,10 +43,10 @@ RSpec.describe Shoryuken::Options do
     end
 
     specify do
-      described_class.add_queue('queue1', 1, 'group1')
-      described_class.add_queue('queue2', 2, 'group2')
+      subject.add_queue('queue1', 1, 'group1')
+      subject.add_queue('queue2', 2, 'group2')
 
-      expect(described_class.ungrouped_queues).to eq(%w[queue1 queue2 queue2])
+      expect(subject.ungrouped_queues).to eq(%w[queue1 queue2 queue2])
     end
   end
 
@@ -52,22 +70,22 @@ RSpec.describe Shoryuken::Options do
 
   describe '.register_worker' do
     it 'registers a worker' do
-      described_class.worker_registry.clear
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+      subject.worker_registry.clear
+      subject.register_worker('default', TestWorker)
+      expect(subject.worker_registry.workers('default')).to eq([TestWorker])
     end
 
     it 'registers a batchable worker' do
-      described_class.worker_registry.clear
+      subject.worker_registry.clear
       TestWorker.get_shoryuken_options['batch'] = true
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+      subject.register_worker('default', TestWorker)
+      expect(subject.worker_registry.workers('default')).to eq([TestWorker])
     end
 
     it 'allows multiple workers' do
-      described_class.worker_registry.clear
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+      subject.worker_registry.clear
+      subject.register_worker('default', TestWorker)
+      expect(subject.worker_registry.workers('default')).to eq([TestWorker])
 
       class Test2Worker
         include Shoryuken::Worker
@@ -77,13 +95,13 @@ RSpec.describe Shoryuken::Options do
         def perform(sqs_msg, body); end
       end
 
-      expect(described_class.worker_registry.workers('default')).to eq([Test2Worker])
+      expect(subject.worker_registry.workers('default')).to eq([Test2Worker])
     end
 
     it 'raises an exception when mixing batchable with non batchable' do
-      described_class.worker_registry.clear
+      subject.worker_registry.clear
       TestWorker.get_shoryuken_options['batch'] = true
-      described_class.register_worker('default', TestWorker)
+      subject.register_worker('default', TestWorker)
 
       expect {
         class BatchableWorker

data/spec/shoryuken/polling/strict_priority_spec.rb

@@ -86,6 +86,14 @@ RSpec.describe Shoryuken::Polling::StrictPriority do
     end
   end
 
+  describe '#delay' do
+    it 'sets delay based on group' do
+      delay_polling = Shoryuken::Polling::StrictPriority.new(queues, 25)
+      expect(delay_polling.delay).to eq(25.0)
+      expect(subject.delay).to eq(1.0)
+    end
+  end
+
   describe '#messages_found' do
     it 'pauses a queue if there are no messages found' do
       # [shoryuken, 2]

data/spec/shoryuken/polling/weighted_round_robin_spec.rb

@@ -96,4 +96,12 @@ RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
       expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
     end
   end
+
+  describe '#delay' do
+    it 'sets delay based on group' do
+      delay_polling = Shoryuken::Polling::WeightedRoundRobin.new(queues, 25)
+      expect(delay_polling.delay).to eq(25.0)
+      expect(subject.delay).to eq(1.0)
+    end
+  end
 end

data/spec/shoryuken/processor_spec.rb

@@ -57,8 +57,8 @@ RSpec.describe Shoryuken::Processor do
 
       context 'server' do
         before do
-          allow(Shoryuken::Options).to receive(:server?).and_return(true)
-          WorkerCalledMiddlewareWorker.instance_variable_set(:@server_chain, nil) # un-memoize middleware
+          allow_any_instance_of(Shoryuken::Options).to receive(:server?).and_return(true)
+          WorkerCalledMiddlewareWorker.instance_variable_set(:@_server_chain, nil) # un-memoize middleware
 
           Shoryuken.configure_server do |config|
             config.server_middleware do |chain|
@@ -85,8 +85,8 @@ RSpec.describe Shoryuken::Processor do
 
       context 'client' do
         before do
-          allow(Shoryuken).to receive(:server?).and_return(false)
-          WorkerCalledMiddlewareWorker.instance_variable_set(:@server_chain, nil) # un-memoize middleware
+          allow_any_instance_of(Shoryuken::Options).to receive(:server?).and_return(false)
+          WorkerCalledMiddlewareWorker.instance_variable_set(:@_server_chain, nil) # un-memoize middleware
 
           Shoryuken.configure_server do |config|
             config.server_middleware do |chain|

data/spec/shoryuken/queue_spec.rb

@@ -26,6 +26,19 @@ RSpec.describe Shoryuken::Queue do
       end
     end
 
+    context 'when a non SQS queue URL' do
+      let(:queue_url) { "http://localhost:4576/queue/#{queue_name}" }
+
+      it 'instantiates by URL and validate the URL' do
+        # See https://github.com/phstc/shoryuken/pull/551
+        expect_any_instance_of(described_class).to receive(:fifo?).and_return(false)
+
+        subject = described_class.new(sqs, queue_url)
+
+        expect(subject.name).to eq(queue_name)
+      end
+    end
+
     context 'when queue name supplied' do
       subject { described_class.new(sqs, queue_name) }
 
@@ -235,7 +248,7 @@ RSpec.describe Shoryuken::Queue do
     end
   end
 
-  describe '#visiblity_timeout' do
+  describe '#visibility_timeout' do
     let(:attribute_response) { double 'Aws::SQS::Types::GetQueueAttributesResponse' }
 
     before do
@@ -247,7 +260,7 @@ RSpec.describe Shoryuken::Queue do
 
     context 'when cache is disabled' do
       specify do
-        Shoryuken.cache_visiblity_timeout = false
+        Shoryuken.cache_visibility_timeout = false
 
         expect(sqs).to(
           receive(:get_queue_attributes).with(queue_url: queue_url, attribute_names: ['All']).and_return(attribute_response).exactly(3).times
@@ -260,7 +273,7 @@ RSpec.describe Shoryuken::Queue do
 
     context 'when cache is enabled' do
       it 'memoizes response' do
-        Shoryuken.cache_visiblity_timeout = true
+        Shoryuken.cache_visibility_timeout = true
 
         expect(sqs).to(
           receive(:get_queue_attributes).with(queue_url: queue_url, attribute_names: ['All']).and_return(attribute_response).once

data/spec/spec_helper.rb

@@ -43,7 +43,7 @@ RSpec.configure do |config|
     Shoryuken.groups.clear
 
     Shoryuken.options[:concurrency] = 1
-    Shoryuken.options[:delay]       = 1
+    Shoryuken.options[:delay]       = 1.0
     Shoryuken.options[:timeout]     = 1
     Shoryuken.options[:daemon]      = nil
     Shoryuken.options[:logfile]     = nil
@@ -61,7 +61,7 @@ RSpec.configure do |config|
 
     Shoryuken.sqs_client_receive_message_opts.clear
 
-    Shoryuken.cache_visiblity_timeout = false
+    Shoryuken.cache_visibility_timeout = false
 
     allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
     allow(Shoryuken).to receive(:active_job?).and_return(false)

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 5.0.0
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-21 00:00:00.000000000 Z
+date: 2019-06-18 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -228,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Shoryuken is a super efficient AWS SQS thread based message processor