sidekiq-limit_fetch 4.3.2 → 4.4.1

data/lib/sidekiq/limit_fetch/queues.rb

@@ -1,159 +1,200 @@
-module Sidekiq::LimitFetch::Queues
-  extend self
+# frozen_string_literal: true
+
+module Sidekiq
+  module LimitFetch
+    module Queues
+      extend self
+
+      THREAD_KEY = :acquired_queues
+
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/PerceivedComplexity
+      def start(capsule_or_options)
+        config = Sidekiq::LimitFetch.post_7? ? capsule_or_options.config : capsule_or_options
+
+        @queues = config[:queues].map do |queue|
+          if queue.is_a? Array
+            queue.first
+          else
+            queue
+          end
+        end.uniq
+        @startup_queues = @queues.dup
+
+        if config[:dynamic].is_a? Hash
+          @dynamic         = true
+          @dynamic_exclude = config[:dynamic][:exclude] || []
+        else
+          @dynamic = config[:dynamic]
+          @dynamic_exclude = []
+        end
 
-  THREAD_KEY = :acquired_queues
+        @limits         = config[:limits] || {}
+        @process_limits = config[:process_limits] || {}
+        @blocks         = config[:blocking] || []
 
-  def start(options)
-    @queues         = options[:queues]
-    @startup_queues = options[:queues].dup
-    @dynamic        = options[:dynamic]
+        config[:strict] ? strict_order! : weighted_order!
 
-    @limits         = options[:limits] || {}
-    @process_limits = options[:process_limits] || {}
-    @blocks         = options[:blocking] || []
+        apply_process_limit_to_queues
+        apply_limit_to_queues
+        apply_blocks_to_queues
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/PerceivedComplexity
+
+      def acquire
+        queues = saved
+        queues ||= Sidekiq::LimitFetch.redis_retryable do
+          selector.acquire(ordered_queues, namespace)
+        end
+        save queues
+        queues.map { |it| "queue:#{it}" }
+      end
 
-    options[:strict] ? strict_order! : weighted_order!
+      def release_except(full_name)
+        queues = restore
+        queues.delete full_name[/queue:(.*)/, 1] if full_name
+        Sidekiq::LimitFetch.redis_retryable do
+          selector.release queues, namespace
+        end
+      end
 
-    apply_process_limit_to_queues
-    apply_limit_to_queues
-    apply_blocks_to_queues
-  end
+      def dynamic?
+        @dynamic
+      end
 
-  def acquire
-    queues = saved
-    queues ||= Sidekiq::LimitFetch.redis_retryable do
-      selector.acquire(ordered_queues, namespace)
-    end
-    save queues
-    queues.map { |it| "queue:#{it}" }
-  end
+      def startup_queue?(queue)
+        @startup_queues.include?(queue)
+      end
 
-  def release_except(full_name)
-    queues = restore
-    queues.delete full_name[/queue:(.*)/, 1] if full_name
-    Sidekiq::LimitFetch.redis_retryable do
-      selector.release queues, namespace
-    end
-  end
+      def dynamic_exclude
+        @dynamic_exclude
+      end
 
-  def dynamic?
-    @dynamic
-  end
+      def add(queues)
+        return unless queues
 
-  def startup_queue?(queue)
-    @startup_queues.include?(queue)
-  end
+        queues.each do |queue|
+          next if @queues.include? queue
 
-  def add(queues)
-    return unless queues
-    queues.each do |queue|
-      unless @queues.include? queue
-        if startup_queue?(queue)
-          apply_process_limit_to_queue(queue)
-          apply_limit_to_queue(queue)
+          if startup_queue?(queue)
+            apply_process_limit_to_queue(queue)
+            apply_limit_to_queue(queue)
+          end
+
+          @queues.push queue
         end
+      end
+
+      def remove(queues)
+        return unless queues
+
+        queues.each do |queue|
+          next unless @queues.include? queue
 
-        @queues.push queue
+          clear_limits_for_queue(queue)
+          @queues.delete queue
+          Sidekiq::Queue.delete_instance(queue)
+        end
       end
-    end
-  end
 
-  def remove(queues)
-    return unless queues
-    queues.each do |queue|
-      if @queues.include? queue
-        clear_limits_for_queue(queue)
-        @queues.delete queue
-        Sidekiq::Queue.delete_instance(queue)
+      def handle(queues)
+        add(queues - @queues)
+        remove(@queues - queues)
       end
-    end
-  end
 
-  def handle(queues)
-    add(queues - @queues)
-    remove(@queues - queues)
-  end
+      # rubocop:disable Lint/NestedMethodDefinition
+      def strict_order!
+        @queues.uniq!
+        def ordered_queues
+          @queues
+        end
+      end
 
-  def strict_order!
-    @queues.uniq!
-    def ordered_queues; @queues end
-  end
+      def weighted_order!
+        def ordered_queues
+          @queues.shuffle.uniq
+        end
+      end
+      # rubocop:enable Lint/NestedMethodDefinition
+
+      def namespace
+        @namespace ||= Sidekiq.redis do |it|
+          if it.respond_to?(:namespace) && it.namespace
+            "#{it.namespace}:"
+          else
+            ''
+          end
+        end
+      end
 
-  def weighted_order!
-    def ordered_queues; @queues.shuffle.uniq end
-  end
+      private
 
-  def namespace
-    @namespace ||= Sidekiq.redis do |it|
-      if it.respond_to?(:namespace) and it.namespace
-        "#{it.namespace}:"
-      else
-        ''
+      def apply_process_limit_to_queues
+        @queues.uniq.each do |queue_name|
+          apply_process_limit_to_queue(queue_name)
+        end
       end
-    end
-  end
-
-  private
 
-  def apply_process_limit_to_queues
-    @queues.uniq.each do |queue_name|
-      apply_process_limit_to_queue(queue_name)
-    end
-  end
+      def apply_process_limit_to_queue(queue_name)
+        queue = Sidekiq::Queue[queue_name]
+        queue.process_limit = @process_limits[queue_name.to_s] || @process_limits[queue_name.to_sym]
+      end
 
-  def apply_process_limit_to_queue(queue_name)
-    queue = Sidekiq::Queue[queue_name]
-    queue.process_limit = @process_limits[queue_name.to_s] || @process_limits[queue_name.to_sym]
-  end
+      def apply_limit_to_queues
+        @queues.uniq.each do |queue_name|
+          apply_limit_to_queue(queue_name)
+        end
+      end
 
-  def apply_limit_to_queues
-    @queues.uniq.each do |queue_name|
-      apply_limit_to_queue(queue_name)
-    end
-  end
+      def apply_limit_to_queue(queue_name)
+        queue = Sidekiq::Queue[queue_name]
 
-  def apply_limit_to_queue(queue_name)
-    queue = Sidekiq::Queue[queue_name]
+        return if queue.limit_changed?
 
-    unless queue.limit_changed?
-      queue.limit = @limits[queue_name.to_s] || @limits[queue_name.to_sym]
-    end
-  end
+        queue.limit = @limits[queue_name.to_s] || @limits[queue_name.to_sym]
+      end
 
-  def apply_blocks_to_queues
-    @queues.uniq.each do |queue_name|
-      Sidekiq::Queue[queue_name].unblock
-    end
+      def apply_blocks_to_queues
+        @queues.uniq.each do |queue_name|
+          Sidekiq::Queue[queue_name].unblock
+        end
 
-    @blocks.to_a.each do |it|
-      if it.is_a? Array
-        it.each {|name| Sidekiq::Queue[name].block_except it }
-      else
-        Sidekiq::Queue[it].block
+        @blocks.to_a.each do |it|
+          if it.is_a? Array
+            it.each { |name| Sidekiq::Queue[name].block_except it }
+          else
+            Sidekiq::Queue[it].block
+          end
+        end
       end
-    end
-  end
 
-  def clear_limits_for_queue(queue_name)
-    queue = Sidekiq::Queue[queue_name]
-    queue.clear_limits
-  end
+      def clear_limits_for_queue(queue_name)
+        queue = Sidekiq::Queue[queue_name]
+        queue.clear_limits
+      end
 
-  def selector
-    Sidekiq::LimitFetch::Global::Selector
-  end
+      def selector
+        Sidekiq::LimitFetch::Global::Selector
+      end
 
-  def saved
-    Thread.current[THREAD_KEY]
-  end
+      def saved
+        Thread.current[THREAD_KEY]
+      end
 
-  def save(queues)
-    Thread.current[THREAD_KEY] = queues
-  end
+      def save(queues)
+        Thread.current[THREAD_KEY] = queues
+      end
 
-  def restore
-    saved || []
-  ensure
-    save nil
+      def restore
+        saved || []
+      ensure
+        save nil
+      end
+    end
   end
 end

data/lib/sidekiq/limit_fetch/unit_of_work.rb

@@ -1,32 +1,36 @@
+# frozen_string_literal: true
+
 module Sidekiq
-  class LimitFetch::UnitOfWork < BasicFetch::UnitOfWork
-    def initialize(queue, job)
-      if post_6_5?
-        super(queue, job, Sidekiq)
-      else
-        super
+  module LimitFetch
+    class UnitOfWork < BasicFetch::UnitOfWork
+      def initialize(queue, job)
+        if post_6_5?
+          super(queue, job, Sidekiq)
+        else
+          super
+        end
+        redis_retryable { Queue[queue_name].increase_busy }
       end
-      redis_retryable { Queue[queue_name].increase_busy }
-    end
 
-    def acknowledge
-      redis_retryable { Queue[queue_name].decrease_busy }
-      redis_retryable { Queue[queue_name].release }
-    end
+      def acknowledge
+        redis_retryable { Queue[queue_name].decrease_busy }
+        redis_retryable { Queue[queue_name].release }
+      end
 
-    def requeue
-      super
-      acknowledge
-    end
+      def requeue
+        super
+        acknowledge
+      end
 
-    private
+      private
 
-    def post_6_5?
-      Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('6.5.0')
-    end
+      def post_6_5?
+        Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('6.5.0')
+      end
 
-    def redis_retryable(&block)
-      Sidekiq::LimitFetch.redis_retryable(&block)
+      def redis_retryable(&block)
+        Sidekiq::LimitFetch.redis_retryable(&block)
+      end
     end
   end
 end

data/lib/sidekiq/limit_fetch.rb

@@ -1,76 +1,95 @@
+# frozen_string_literal: true
+
 require 'forwardable'
 require 'sidekiq'
 require 'sidekiq/manager'
 require 'sidekiq/api'
 
-module Sidekiq::LimitFetch
-  autoload :UnitOfWork, 'sidekiq/limit_fetch/unit_of_work'
+module Sidekiq
+  module LimitFetch
+    autoload :UnitOfWork, 'sidekiq/limit_fetch/unit_of_work'
 
-  require_relative 'limit_fetch/instances'
-  require_relative 'limit_fetch/queues'
-  require_relative 'limit_fetch/global/semaphore'
-  require_relative 'limit_fetch/global/selector'
-  require_relative 'limit_fetch/global/monitor'
-  require_relative 'extensions/queue'
-  require_relative 'extensions/manager'
+    require_relative 'limit_fetch/instances'
+    require_relative 'limit_fetch/queues'
+    require_relative 'limit_fetch/global/semaphore'
+    require_relative 'limit_fetch/global/selector'
+    require_relative 'limit_fetch/global/monitor'
+    require_relative 'extensions/queue'
+    require_relative 'extensions/manager'
 
-  TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
+    TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
 
-  extend self
+    extend self
 
-  def new(_)
-    self
-  end
+    def post_7?
+      @post_7 ||= Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('7.0.0')
+    end
 
-  def retrieve_work
-    queue, job = redis_brpop(Queues.acquire)
-    Queues.release_except(queue)
-    UnitOfWork.new(queue, job) if job
-  end
+    def post_6_5?
+      @post_6_5 ||= Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('6.5.0')
+    end
 
-  def config
-    # Post 6.5, Sidekiq.options is deprecated and replaced with passing Sidekiq directly
-    post_6_5? ? Sidekiq : Sidekiq.options
-  end
+    RedisBaseConnectionError = post_7? ? RedisClient::ConnectionError : Redis::BaseConnectionError
+    RedisCommandError = post_7? ? RedisClient::CommandError : Redis::CommandError
 
-  # Backwards compatibility for sidekiq v6.1.0
-  # @see https://github.com/mperham/sidekiq/pull/4602
-  def bulk_requeue(*args)
-    if Sidekiq::BasicFetch.respond_to?(:bulk_requeue) # < 6.1.0
-      Sidekiq::BasicFetch.bulk_requeue(*args)
-    else # 6.1.0+
-      Sidekiq::BasicFetch.new(config).bulk_requeue(*args)
+    def new(_)
+      self
     end
-  end
 
-  def redis_retryable
-    yield
-  rescue Redis::BaseConnectionError
-    sleep TIMEOUT
-    retry
-  rescue Redis::CommandError => error
-    # If Redis was restarted and is still loading its snapshot,
-    # then we should treat this as a temporary connection error too.
-    if error.message =~ /^LOADING/
+    def retrieve_work
+      queue, job = redis_brpop(Queues.acquire)
+      Queues.release_except(queue)
+      UnitOfWork.new(queue, job) if job
+    end
+
+    def config
+      # Post 6.5, Sidekiq.options is deprecated and replaced with passing Sidekiq directly
+      post_6_5? ? Sidekiq : Sidekiq.options
+    end
+
+    # Backwards compatibility for sidekiq v6.1.0
+    # @see https://github.com/mperham/sidekiq/pull/4602
+    def bulk_requeue(*args)
+      if Sidekiq::BasicFetch.respond_to?(:bulk_requeue) # < 6.1.0
+        Sidekiq::BasicFetch.bulk_requeue(*args)
+      else # 6.1.0+
+        Sidekiq::BasicFetch.new(post_7? ? Sidekiq.default_configuration.default_capsule : config).bulk_requeue(*args)
+      end
+    end
+
+    def redis_retryable
+      yield
+    rescue RedisBaseConnectionError
       sleep TIMEOUT
       retry
-    else
-      raise
-    end
-  end
+    rescue RedisCommandError => e
+      # If Redis was restarted and is still loading its snapshot,
+      # then we should treat this as a temporary connection error too.
+      raise unless e.message =~ /^LOADING/
 
-  private
+      sleep TIMEOUT
+      retry
+    end
 
-  def post_6_5?
-    @post_6_5 ||= Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('6.5.0')
-  end
+    private
 
-  def redis_brpop(queues)
-    if queues.empty?
-      sleep TIMEOUT  # there are no queues to handle, so lets sleep
-      []             # and return nothing
-    else
-      redis_retryable { Sidekiq.redis { |it| it.brpop *queues, timeout: TIMEOUT } }
+    # rubocop:disable Metrics/MethodLength
+    def redis_brpop(queues)
+      if queues.empty?
+        sleep TIMEOUT  # there are no queues to handle, so lets sleep
+        []             # and return nothing
+      else
+        redis_retryable do
+          Sidekiq.redis do |it|
+            if post_7?
+              it.blocking_call(false, 'brpop', *queues, TIMEOUT)
+            else
+              it.brpop(*queues, timeout: TIMEOUT)
+            end
+          end
+        end
+      end
     end
+    # rubocop:enable Metrics/MethodLength
   end
 end

data/lib/sidekiq-limit_fetch.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require_relative 'sidekiq/limit_fetch'

data/sidekiq-limit_fetch.gemspec

@@ -1,25 +1,32 @@
+# frozen_string_literal: true
+
+require 'English'
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '4.3.2'
+  gem.version       = '4.4.1'
   gem.license       = 'MIT'
-  gem.authors       = ['Dean Perry', 'brainopia']
+  gem.authors       = ['Dean Perry']
   gem.email         = 'dean@deanpcmad.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'
   gem.homepage      = 'https://github.com/deanpcmad/sidekiq-limit_fetch'
-  gem.description   = "Sidekiq strategy to restrict number of workers which are able to run specified queues simultaneously."
+  gem.description   = 'Sidekiq strategy to restrict number of workers which are able to run specified ' \
+                      'queues simultaneously.'
 
-  gem.metadata["homepage_uri"] = gem.homepage
-  gem.metadata["source_code_uri"] = "https://github.com/deanpcmad/sidekiq-limit_fetch"
-  gem.metadata["changelog_uri"] = "https://github.com/deanpcmad/sidekiq-limit_fetch/blob/master/CHANGELOG.md"
+  gem.metadata['homepage_uri'] = gem.homepage
+  gem.metadata['source_code_uri'] = 'https://github.com/deanpcmad/sidekiq-limit_fetch'
+  gem.metadata['changelog_uri'] = 'https://github.com/deanpcmad/sidekiq-limit_fetch/blob/master/CHANGELOG.md'
+  gem.metadata['rubygems_mfa_required'] = 'true'
 
-  gem.files         = `git ls-files`.split($/)
-  gem.test_files    = gem.files.grep %r{^spec/}
-  gem.require_paths = %w(lib)
+  gem.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  gem.require_paths = %w[lib]
 
-  gem.add_dependency 'sidekiq', '>= 4'
-  gem.add_dependency 'redis', '>= 4.6.0'
-  gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2'
+  gem.required_ruby_version = '>= 2.7.0'
+
+  gem.add_dependency 'sidekiq', '>= 6'
   gem.add_development_dependency 'appraisal'
-  gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rubocop'
+  gem.add_development_dependency 'simplecov'
 end

data/spec/sidekiq/extensions/manager_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+RSpec.describe Sidekiq::Manager do
+  let(:capsule_or_options) do
+    if Sidekiq::LimitFetch.post_7?
+      Sidekiq.default_configuration.default_capsule
+    elsif Sidekiq::LimitFetch.post_6_5?
+      Sidekiq
+    else
+      Sidekiq.options
+    end
+  end
+
+  it 'can be instantiated' do
+    expect(described_class).to be < Sidekiq::Manager::InitLimitFetch
+    manager = described_class.new(capsule_or_options)
+    expect(manager).to respond_to(:start)
+  end
+end

data/spec/sidekiq/extensions/queue_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 RSpec.describe Sidekiq::Queue do
   context 'singleton' do
     shared_examples :constructor do