sidekiq-limit_fetch 3.3.1 → 4.1.0

data/gemfiles/sidekiq_6.2.gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: ..
+  specs:
+    sidekiq-limit_fetch (3.4.0)
+      redis (>= 4.6.0)
+      sidekiq (>= 4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    connection_pool (2.2.5)
+    diff-lcs (1.5.0)
+    rack (2.2.3)
+    rake (13.0.6)
+    redis (4.6.0)
+    redis-namespace (1.8.2)
+      redis (>= 3.0.4)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    sidekiq (6.2.1)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    thor (1.2.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  rake
+  redis-namespace (~> 1.5, >= 1.5.2)
+  rspec
+  sidekiq (~> 6.2.0)
+  sidekiq-limit_fetch!
+
+BUNDLED WITH
+   2.3.6

data/gemfiles/sidekiq_6.3.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 6.3.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_6.3.gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: ..
+  specs:
+    sidekiq-limit_fetch (3.4.0)
+      redis (>= 4.6.0)
+      sidekiq (>= 4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    connection_pool (2.2.5)
+    diff-lcs (1.5.0)
+    rack (2.2.3)
+    rake (13.0.6)
+    redis (4.6.0)
+    redis-namespace (1.8.2)
+      redis (>= 3.0.4)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    sidekiq (6.3.1)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    thor (1.2.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  rake
+  redis-namespace (~> 1.5, >= 1.5.2)
+  rspec
+  sidekiq (~> 6.3.0)
+  sidekiq-limit_fetch!
+
+BUNDLED WITH
+   2.3.6

data/gemfiles/sidekiq_6.4.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 6.4.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_6.4.gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: ..
+  specs:
+    sidekiq-limit_fetch (3.4.0)
+      redis (>= 4.6.0)
+      sidekiq (>= 4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    connection_pool (2.2.5)
+    diff-lcs (1.5.0)
+    rack (2.2.3)
+    rake (13.0.6)
+    redis (4.6.0)
+    redis-namespace (1.8.2)
+      redis (>= 3.0.4)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    sidekiq (6.4.1)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    thor (1.2.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  rake
+  redis-namespace (~> 1.5, >= 1.5.2)
+  rspec
+  sidekiq (~> 6.4.0)
+  sidekiq-limit_fetch!
+
+BUNDLED WITH
+   2.3.6

data/gemfiles/sidekiq_master.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", github: "mperham/sidekiq"
+
+gemspec path: "../"

data/gemfiles/sidekiq_master.gemfile.lock

@@ -0,0 +1,58 @@
+GIT
+  remote: https://github.com/mperham/sidekiq.git
+  revision: cf7b067c89ae3b1303e35d29408099cf40991f6d
+  specs:
+    sidekiq (6.4.2)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+
+PATH
+  remote: ..
+  specs:
+    sidekiq-limit_fetch (3.4.0)
+      redis (>= 4.6.0)
+      sidekiq (>= 4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    connection_pool (2.2.5)
+    diff-lcs (1.5.0)
+    rack (2.2.3)
+    rake (13.0.6)
+    redis (4.6.0)
+    redis-namespace (1.8.2)
+      redis (>= 3.0.4)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    thor (1.2.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  rake
+  redis-namespace (~> 1.5, >= 1.5.2)
+  rspec
+  sidekiq!
+  sidekiq-limit_fetch!
+
+BUNDLED WITH
+   2.3.6

data/lib/sidekiq/extensions/queue.rb

@@ -14,7 +14,8 @@ module Sidekiq
       :probed,        :busy,
       :increase_busy, :decrease_busy,
       :local_busy?,   :explain,
-      :remove_locks_except!
+      :remove_locks_except!,
+      :clear_limits
 
     def lock
       @lock ||= LimitFetch::Global::Semaphore.new name

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -11,7 +11,7 @@ module Sidekiq::LimitFetch::Global
       Thread.new do
         loop do
           Sidekiq::LimitFetch.redis_retryable do
-            add_dynamic_queues
+            handle_dynamic_queues
             update_heartbeat ttl
             invalidate_old_processes
           end
@@ -37,19 +37,19 @@ module Sidekiq::LimitFetch::Global
       end
     end
 
-    def add_dynamic_queues
+    def handle_dynamic_queues
       queues = Sidekiq::LimitFetch::Queues
-      queues.add Sidekiq::Queue.all.map(&:name) if queues.dynamic?
+      queues.handle Sidekiq::Queue.all.map(&:name) if queues.dynamic?
     end
 
     private
 
     def update_heartbeat(ttl)
       Sidekiq.redis do |it|
-        it.multi do
-          it.set heartbeat_key, true
-          it.sadd PROCESS_SET, Selector.uuid
-          it.expire heartbeat_key, ttl
+        it.multi do |pipeline|
+          pipeline.set heartbeat_key, true
+          pipeline.sadd PROCESS_SET, Selector.uuid
+          pipeline.expire heartbeat_key, ttl
         end
       end
     end

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -109,6 +109,14 @@ module Sidekiq::LimitFetch::Global
       redis {|it| it.get "#{PREFIX}:block:#@name" }
     end
 
+    def clear_limits
+      redis do |it|
+        %w(block busy limit pause probed process_limit).each do |key|
+          it.del "#{PREFIX}:#{key}:#@name"
+        end
+      end
+    end
+
     def increase_local_busy
       @lock.synchronize { @local_busy += 1 }
     end

data/lib/sidekiq/limit_fetch/instances.rb

@@ -16,4 +16,8 @@ module Sidekiq::LimitFetch::Instances
   def reset_instances!
     @instances = {}
   end
+
+  def delete_instance(name)
+    @instances.delete [name]
+  end
 end

data/lib/sidekiq/limit_fetch/queues.rb

@@ -4,26 +4,33 @@ module Sidekiq::LimitFetch::Queues
   THREAD_KEY = :acquired_queues
 
   def start(options)
-    @queues    = options[:queues]
-    @dynamic   = options[:dynamic]
+    @queues         = options[:queues]
+    @startup_queues = options[:queues].dup
+    @dynamic        = options[:dynamic]
+
+    @limits         = options[:limits] || {}
+    @process_limits = options[:process_limits] || {}
+    @blocks         = options[:blocking] || []
 
     options[:strict] ? strict_order! : weighted_order!
 
-    set :process_limit, options[:process_limits]
-    set :limit, options[:limits]
-    set_blocks options[:blocking]
+    apply_process_limit_to_queues
+    apply_limit_to_queues
+    apply_blocks_to_queues
   end
 
   def acquire
-    selector.acquire(ordered_queues, namespace)
-      .tap {|it| save it }
-      .map {|it| "queue:#{it}" }
+    queues = saved
+    queues ||= Sidekiq::LimitFetch.redis_retryable do
+      selector.acquire(ordered_queues, namespace)
+    end
+    save queues
+    queues.map { |it| "queue:#{it}" }
   end
 
   def release_except(full_name)
     queues = restore
     queues.delete full_name[/queue:(.*)/, 1] if full_name
-
     Sidekiq::LimitFetch.redis_retryable do
       selector.release queues, namespace
     end
@@ -33,12 +40,40 @@ module Sidekiq::LimitFetch::Queues
     @dynamic
   end
 
+  def startup_queue?(queue)
+    @startup_queues.include?(queue)
+  end
+
   def add(queues)
+    return unless queues
     queues.each do |queue|
-      @queues.push queue unless @queues.include? queue
+      unless @queues.include? queue
+        if startup_queue?(queue)
+          apply_process_limit_to_queue(queue)
+          apply_limit_to_queue(queue)
+        end
+
+        @queues.push queue
+      end
     end
   end
 
+  def remove(queues)
+    return unless queues
+    queues.each do |queue|
+      if @queues.include? queue
+        clear_limits_for_queue(queue)
+        @queues.delete queue
+        Sidekiq::Queue.delete_instance(queue)
+      end
+    end
+  end
+
+  def handle(queues)
+    add(queues - @queues)
+    remove(@queues - queues)
+  end
+
   def strict_order!
     @queues.uniq!
     def ordered_queues; @queues end
@@ -60,22 +95,37 @@ module Sidekiq::LimitFetch::Queues
 
   private
 
-  def selector
-    Sidekiq::LimitFetch::Global::Selector
+  def apply_process_limit_to_queues
+    @queues.uniq.each do |queue_name|
+      apply_process_limit_to_queue(queue_name)
+    end
   end
 
-  def set(limit_type, limits)
-    limits ||= {}
-    each_queue do |queue|
-      limit = limits[queue.name.to_s] || limits[queue.name.to_sym]
-      queue.send "#{limit_type}=", limit unless queue.limit_changed?
+  def apply_process_limit_to_queue(queue_name)
+    queue = Sidekiq::Queue[queue_name]
+    queue.process_limit = @process_limits[queue_name.to_s] || @process_limits[queue_name.to_sym]
+  end
+
+  def apply_limit_to_queues
+    @queues.uniq.each do |queue_name|
+      apply_limit_to_queue(queue_name)
     end
   end
 
-  def set_blocks(blocks)
-    each_queue(&:unblock)
+  def apply_limit_to_queue(queue_name)
+    queue = Sidekiq::Queue[queue_name]
+
+    unless queue.limit_changed?
+      queue.limit = @limits[queue_name.to_s] || @limits[queue_name.to_sym]
+    end
+  end
 
-    blocks.to_a.each do |it|
+  def apply_blocks_to_queues
+    @queues.uniq.each do |queue_name|
+      Sidekiq::Queue[queue_name].unblock
+    end
+
+    @blocks.to_a.each do |it|
       if it.is_a? Array
         it.each {|name| Sidekiq::Queue[name].block_except it }
       else
@@ -84,17 +134,26 @@ module Sidekiq::LimitFetch::Queues
     end
   end
 
+  def clear_limits_for_queue(queue_name)
+    queue = Sidekiq::Queue[queue_name]
+    queue.clear_limits
+  end
+
+  def selector
+    Sidekiq::LimitFetch::Global::Selector
+  end
+
+  def saved
+    Thread.current[THREAD_KEY]
+  end
+
   def save(queues)
     Thread.current[THREAD_KEY] = queues
   end
 
   def restore
-    Thread.current[THREAD_KEY] || []
+    saved || []
   ensure
-    Thread.current[THREAD_KEY] = nil
-  end
-
-  def each_queue
-    @queues.uniq.each {|it| yield Sidekiq::Queue[it] }
+    save nil
   end
 end

data/lib/sidekiq/limit_fetch.rb

@@ -14,6 +14,8 @@ module Sidekiq::LimitFetch
   require_relative 'extensions/queue'
   require_relative 'extensions/manager'
 
+  TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
+
   extend self
 
   def new(_)
@@ -26,21 +28,34 @@ module Sidekiq::LimitFetch
     UnitOfWork.new(queue, job) if job
   end
 
+  # Backwards compatibility for sidekiq v6.1.0
+  # @see https://github.com/mperham/sidekiq/pull/4602
   def bulk_requeue(*args)
-    Sidekiq::BasicFetch.bulk_requeue(*args)
+    if Sidekiq::BasicFetch.respond_to?(:bulk_requeue) # < 6.1.0
+      Sidekiq::BasicFetch.bulk_requeue(*args)
+    else # 6.1.0+
+      Sidekiq::BasicFetch.new(Sidekiq.options).bulk_requeue(*args)
+    end
   end
 
   def redis_retryable
     yield
   rescue Redis::BaseConnectionError
-    sleep 1
+    sleep TIMEOUT
     retry
+  rescue Redis::CommandError => error
+    # If Redis was restarted and is still loading its snapshot,
+    # then we should treat this as a temporary connection error too.
+    if error.message =~ /^LOADING/
+      sleep TIMEOUT
+      retry
+    else
+      raise
+    end
   end
 
   private
 
-  TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
-
   def redis_brpop(queues)
     if queues.empty?
       sleep TIMEOUT  # there are no queues to handle, so lets sleep

data/sidekiq-limit_fetch.gemspec

@@ -1,22 +1,25 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '3.3.1'
+  gem.version       = '4.1.0'
   gem.license       = 'MIT'
-  gem.authors       = 'brainopia'
-  gem.email         = 'brainopia@evilmartians.com'
+  gem.authors       = ['Dean Perry', 'brainopia']
+  gem.email         = 'dean@deanpcmad.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'
-  gem.homepage      = 'https://github.com/brainopia/sidekiq-limit_fetch'
-  gem.description   = <<-DESCRIPTION
-    Sidekiq strategy to restrict number of workers
-    which are able to run specified queues simultaneously.
-  DESCRIPTION
+  gem.homepage      = 'https://github.com/deanpcmad/sidekiq-limit_fetch'
+  gem.description   = "Sidekiq strategy to restrict number of workers which are able to run specified queues simultaneously."
+
+  gem.metadata["homepage_uri"] = gem.homepage
+  gem.metadata["source_code_uri"] = "https://github.com/deanpcmad/sidekiq-limit_fetch"
+  gem.metadata["changelog_uri"] = "https://github.com/deanpcmad/sidekiq-limit_fetch/blob/master/CHANGELOG.md"
 
   gem.files         = `git ls-files`.split($/)
   gem.test_files    = gem.files.grep %r{^spec/}
   gem.require_paths = %w(lib)
 
   gem.add_dependency 'sidekiq', '>= 4'
+  gem.add_dependency 'redis', '>= 4.6.0'
   gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2'
+  gem.add_development_dependency 'appraisal'
   gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'rake'
 end

data/spec/sidekiq/limit_fetch/queues_spec.rb

@@ -15,61 +15,72 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
 
   before { subject.start options }
 
+  def in_thread(&block)
+    thr = Thread.new(&block)
+    thr.join
+  end
+
   it 'should acquire queues' do
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
   end
 
   it 'should acquire dynamically blocking queues' do
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
 
     Sidekiq::Queue['queue1'].block
 
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 2
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
   end
 
   it 'should block except given queues' do
     Sidekiq::Queue['queue1'].block_except 'queue2'
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
 
     Sidekiq::Queue['queue1'].block_except 'queue404'
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 2
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
   end
 
   it 'should release queues' do
-    subject.acquire
-    subject.release_except nil
+    in_thread {
+      subject.acquire
+      subject.release_except nil
+    }
     expect(Sidekiq::Queue['queue1'].probed).to eq 0
     expect(Sidekiq::Queue['queue2'].probed).to eq 0
   end
 
   it 'should release queues except selected' do
-    subject.acquire
-    subject.release_except 'queue:queue1'
+    in_thread {
+      subject.acquire
+      subject.release_except 'queue:queue1'
+    }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 0
   end
 
   it 'should release when no queues was acquired' do
     queues.each {|name| Sidekiq::Queue[name].pause }
-    subject.acquire
-    expect { subject.release_except nil }.not_to raise_exception
+    in_thread {
+      subject.acquire
+      expect { subject.release_except nil }.not_to raise_exception
+    }
   end
 
   context 'blocking' do
     let(:blocking) { %w(queue1) }
 
     it 'should acquire blocking queues' do
-      3.times { subject.acquire }
+      3.times { in_thread { subject.acquire  } }
       expect(Sidekiq::Queue['queue1'].probed).to eq 3
       expect(Sidekiq::Queue['queue2'].probed).to eq 1
     end

data/spec/sidekiq/limit_fetch_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe Sidekiq::LimitFetch do
   let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
 
   before do
-    subject::Queues.start options 
+    subject::Queues.start options
 
     Sidekiq.redis do |it|
       it.del 'queue:queue1'

data/spec/spec_helper.rb

@@ -11,13 +11,13 @@ RSpec.configure do |config|
     Sidekiq::Queue.reset_instances!
     Sidekiq.redis do |it|
       clean_redis = ->(queue) do
-        it.pipelined do
-          it.del "limit_fetch:limit:#{queue}"
-          it.del "limit_fetch:process_limit:#{queue}"
-          it.del "limit_fetch:busy:#{queue}"
-          it.del "limit_fetch:probed:#{queue}"
-          it.del "limit_fetch:pause:#{queue}"
-          it.del "limit_fetch:block:#{queue}"
+        it.pipelined do |pipeline|
+          pipeline.del "limit_fetch:limit:#{queue}"
+          pipeline.del "limit_fetch:process_limit:#{queue}"
+          pipeline.del "limit_fetch:busy:#{queue}"
+          pipeline.del "limit_fetch:probed:#{queue}"
+          pipeline.del "limit_fetch:pause:#{queue}"
+          pipeline.del "limit_fetch:block:#{queue}"
         end
       end