sidekiq-limit_fetch 3.4.0 → 4.2.0

data/gemfiles/sidekiq_6.5.gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: ..
+  specs:
+    sidekiq-limit_fetch (4.1.0)
+      redis (>= 4.6.0)
+      sidekiq (>= 4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    connection_pool (2.2.5)
+    diff-lcs (1.5.0)
+    rack (2.2.3.1)
+    rake (13.0.6)
+    redis (4.6.0)
+    redis-namespace (1.8.2)
+      redis (>= 3.0.4)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    sidekiq (6.5.0)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    thor (1.2.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  rake
+  redis-namespace (~> 1.5, >= 1.5.2)
+  rspec
+  sidekiq (~> 6.5.0)
+  sidekiq-limit_fetch!
+
+BUNDLED WITH
+   2.3.5

data/gemfiles/sidekiq_master.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", github: "mperham/sidekiq"
+
+gemspec path: "../"

data/gemfiles/sidekiq_master.gemfile.lock

@@ -0,0 +1,58 @@
+GIT
+  remote: https://github.com/mperham/sidekiq.git
+  revision: 277ea1888f25917e9dde8d00bdb866f86a55691f
+  specs:
+    sidekiq (6.5.0)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+
+PATH
+  remote: ..
+  specs:
+    sidekiq-limit_fetch (4.1.0)
+      redis (>= 4.6.0)
+      sidekiq (>= 4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    connection_pool (2.2.5)
+    diff-lcs (1.5.0)
+    rack (2.2.3.1)
+    rake (13.0.6)
+    redis (4.6.0)
+    redis-namespace (1.8.2)
+      redis (>= 3.0.4)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    thor (1.2.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  rake
+  redis-namespace (~> 1.5, >= 1.5.2)
+  rspec
+  sidekiq!
+  sidekiq-limit_fetch!
+
+BUNDLED WITH
+   2.3.14

data/lib/sidekiq/extensions/manager.rb

@@ -6,7 +6,8 @@ class Sidekiq::Manager
     end
 
     def start
-      Sidekiq::LimitFetch::Queues.start options
+      # In sidekiq 6.5.0 the variable @options has been renamed to @config
+      Sidekiq::LimitFetch::Queues.start @options || @config
       Sidekiq::LimitFetch::Global::Monitor.start!
       super
     end

data/lib/sidekiq/extensions/queue.rb

@@ -14,7 +14,8 @@ module Sidekiq
       :probed,        :busy,
       :increase_busy, :decrease_busy,
       :local_busy?,   :explain,
-      :remove_locks_except!
+      :remove_locks_except!,
+      :clear_limits
 
     def lock
       @lock ||= LimitFetch::Global::Semaphore.new name

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -11,7 +11,7 @@ module Sidekiq::LimitFetch::Global
       Thread.new do
         loop do
           Sidekiq::LimitFetch.redis_retryable do
-            add_dynamic_queues
+            handle_dynamic_queues
             update_heartbeat ttl
             invalidate_old_processes
           end
@@ -37,19 +37,19 @@ module Sidekiq::LimitFetch::Global
       end
     end
 
-    def add_dynamic_queues
+    def handle_dynamic_queues
       queues = Sidekiq::LimitFetch::Queues
-      queues.add Sidekiq::Queue.all.map(&:name) if queues.dynamic?
+      queues.handle Sidekiq::Queue.all.map(&:name) if queues.dynamic?
     end
 
     private
 
     def update_heartbeat(ttl)
       Sidekiq.redis do |it|
-        it.multi do
-          it.set heartbeat_key, true
-          it.sadd PROCESS_SET, Selector.uuid
-          it.expire heartbeat_key, ttl
+        it.multi do |pipeline|
+          pipeline.set heartbeat_key, true
+          pipeline.sadd PROCESS_SET, Selector.uuid
+          pipeline.expire heartbeat_key, ttl
         end
       end
     end

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -109,6 +109,14 @@ module Sidekiq::LimitFetch::Global
       redis {|it| it.get "#{PREFIX}:block:#@name" }
     end
 
+    def clear_limits
+      redis do |it|
+        %w(block busy limit pause probed process_limit).each do |key|
+          it.del "#{PREFIX}:#{key}:#@name"
+        end
+      end
+    end
+
     def increase_local_busy
       @lock.synchronize { @local_busy += 1 }
     end

data/lib/sidekiq/limit_fetch/instances.rb

@@ -16,4 +16,8 @@ module Sidekiq::LimitFetch::Instances
   def reset_instances!
     @instances = {}
   end
+
+  def delete_instance(name)
+    @instances.delete [name]
+  end
 end

data/lib/sidekiq/limit_fetch/queues.rb

@@ -5,6 +5,7 @@ module Sidekiq::LimitFetch::Queues
 
   def start(options)
     @queues         = options[:queues]
+    @startup_queues = options[:queues].dup
     @dynamic        = options[:dynamic]
 
     @limits         = options[:limits] || {}
@@ -19,15 +20,17 @@ module Sidekiq::LimitFetch::Queues
   end
 
   def acquire
-    selector.acquire(ordered_queues, namespace)
-      .tap {|it| save it }
-      .map {|it| "queue:#{it}" }
+    queues = saved
+    queues ||= Sidekiq::LimitFetch.redis_retryable do
+      selector.acquire(ordered_queues, namespace)
+    end
+    save queues
+    queues.map { |it| "queue:#{it}" }
   end
 
   def release_except(full_name)
     queues = restore
     queues.delete full_name[/queue:(.*)/, 1] if full_name
-
     Sidekiq::LimitFetch.redis_retryable do
       selector.release queues, namespace
     end
@@ -37,17 +40,40 @@ module Sidekiq::LimitFetch::Queues
     @dynamic
   end
 
+  def startup_queue?(queue)
+    @startup_queues.include?(queue)
+  end
+
   def add(queues)
+    return unless queues
     queues.each do |queue|
       unless @queues.include? queue
-        apply_process_limit_to_queue(queue)
-        apply_limit_to_queue(queue)
+        if startup_queue?(queue)
+          apply_process_limit_to_queue(queue)
+          apply_limit_to_queue(queue)
+        end
 
         @queues.push queue
       end
     end
   end
 
+  def remove(queues)
+    return unless queues
+    queues.each do |queue|
+      if @queues.include? queue
+        clear_limits_for_queue(queue)
+        @queues.delete queue
+        Sidekiq::Queue.delete_instance(queue)
+      end
+    end
+  end
+
+  def handle(queues)
+    add(queues - @queues)
+    remove(@queues - queues)
+  end
+
   def strict_order!
     @queues.uniq!
     def ordered_queues; @queues end
@@ -108,17 +134,26 @@ module Sidekiq::LimitFetch::Queues
     end
   end
 
+  def clear_limits_for_queue(queue_name)
+    queue = Sidekiq::Queue[queue_name]
+    queue.clear_limits
+  end
+
   def selector
     Sidekiq::LimitFetch::Global::Selector
   end
 
+  def saved
+    Thread.current[THREAD_KEY]
+  end
+
   def save(queues)
     Thread.current[THREAD_KEY] = queues
   end
 
   def restore
-    Thread.current[THREAD_KEY] || []
+    saved || []
   ensure
-    Thread.current[THREAD_KEY] = nil
+    save nil
   end
 end

data/lib/sidekiq/limit_fetch.rb

@@ -14,6 +14,8 @@ module Sidekiq::LimitFetch
   require_relative 'extensions/queue'
   require_relative 'extensions/manager'
 
+  TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
+
   extend self
 
   def new(_)
@@ -26,21 +28,34 @@ module Sidekiq::LimitFetch
     UnitOfWork.new(queue, job) if job
   end
 
+  # Backwards compatibility for sidekiq v6.1.0
+  # @see https://github.com/mperham/sidekiq/pull/4602
   def bulk_requeue(*args)
-    Sidekiq::BasicFetch.bulk_requeue(*args)
+    if Sidekiq::BasicFetch.respond_to?(:bulk_requeue) # < 6.1.0
+      Sidekiq::BasicFetch.bulk_requeue(*args)
+    else # 6.1.0+
+      Sidekiq::BasicFetch.new(Sidekiq.options).bulk_requeue(*args)
+    end
   end
 
   def redis_retryable
     yield
   rescue Redis::BaseConnectionError
-    sleep 1
+    sleep TIMEOUT
     retry
+  rescue Redis::CommandError => error
+    # If Redis was restarted and is still loading its snapshot,
+    # then we should treat this as a temporary connection error too.
+    if error.message =~ /^LOADING/
+      sleep TIMEOUT
+      retry
+    else
+      raise
+    end
   end
 
   private
 
-  TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
-
   def redis_brpop(queues)
     if queues.empty?
       sleep TIMEOUT  # there are no queues to handle, so lets sleep

data/sidekiq-limit_fetch.gemspec

@@ -1,22 +1,25 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '3.4.0'
+  gem.version       = '4.2.0'
   gem.license       = 'MIT'
-  gem.authors       = 'brainopia'
-  gem.email         = 'brainopia@evilmartians.com'
+  gem.authors       = ['Dean Perry', 'brainopia']
+  gem.email         = 'dean@deanpcmad.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'
-  gem.homepage      = 'https://github.com/brainopia/sidekiq-limit_fetch'
-  gem.description   = <<-DESCRIPTION
-    Sidekiq strategy to restrict number of workers
-    which are able to run specified queues simultaneously.
-  DESCRIPTION
+  gem.homepage      = 'https://github.com/deanpcmad/sidekiq-limit_fetch'
+  gem.description   = "Sidekiq strategy to restrict number of workers which are able to run specified queues simultaneously."
+
+  gem.metadata["homepage_uri"] = gem.homepage
+  gem.metadata["source_code_uri"] = "https://github.com/deanpcmad/sidekiq-limit_fetch"
+  gem.metadata["changelog_uri"] = "https://github.com/deanpcmad/sidekiq-limit_fetch/blob/master/CHANGELOG.md"
 
   gem.files         = `git ls-files`.split($/)
   gem.test_files    = gem.files.grep %r{^spec/}
   gem.require_paths = %w(lib)
 
   gem.add_dependency 'sidekiq', '>= 4'
+  gem.add_dependency 'redis', '>= 4.6.0'
   gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2'
+  gem.add_development_dependency 'appraisal'
   gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'rake'
 end

data/spec/sidekiq/limit_fetch/queues_spec.rb

@@ -15,61 +15,72 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
 
   before { subject.start options }
 
+  def in_thread(&block)
+    thr = Thread.new(&block)
+    thr.join
+  end
+
   it 'should acquire queues' do
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
   end
 
   it 'should acquire dynamically blocking queues' do
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
 
     Sidekiq::Queue['queue1'].block
 
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 2
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
   end
 
   it 'should block except given queues' do
     Sidekiq::Queue['queue1'].block_except 'queue2'
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
 
     Sidekiq::Queue['queue1'].block_except 'queue404'
-    subject.acquire
+    in_thread { subject.acquire }
     expect(Sidekiq::Queue['queue1'].probed).to eq 2
     expect(Sidekiq::Queue['queue2'].probed).to eq 1
   end
 
   it 'should release queues' do
-    subject.acquire
-    subject.release_except nil
+    in_thread {
+      subject.acquire
+      subject.release_except nil
+    }
     expect(Sidekiq::Queue['queue1'].probed).to eq 0
     expect(Sidekiq::Queue['queue2'].probed).to eq 0
   end
 
   it 'should release queues except selected' do
-    subject.acquire
-    subject.release_except 'queue:queue1'
+    in_thread {
+      subject.acquire
+      subject.release_except 'queue:queue1'
+    }
     expect(Sidekiq::Queue['queue1'].probed).to eq 1
     expect(Sidekiq::Queue['queue2'].probed).to eq 0
   end
 
   it 'should release when no queues was acquired' do
     queues.each {|name| Sidekiq::Queue[name].pause }
-    subject.acquire
-    expect { subject.release_except nil }.not_to raise_exception
+    in_thread {
+      subject.acquire
+      expect { subject.release_except nil }.not_to raise_exception
+    }
   end
 
   context 'blocking' do
     let(:blocking) { %w(queue1) }
 
     it 'should acquire blocking queues' do
-      3.times { subject.acquire }
+      3.times { in_thread { subject.acquire  } }
       expect(Sidekiq::Queue['queue1'].probed).to eq 3
       expect(Sidekiq::Queue['queue2'].probed).to eq 1
     end

data/spec/sidekiq/limit_fetch_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe Sidekiq::LimitFetch do
   let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
 
   before do
-    subject::Queues.start options 
+    subject::Queues.start options
 
     Sidekiq.redis do |it|
       it.del 'queue:queue1'

data/spec/spec_helper.rb

@@ -11,13 +11,13 @@ RSpec.configure do |config|
     Sidekiq::Queue.reset_instances!
     Sidekiq.redis do |it|
       clean_redis = ->(queue) do
-        it.pipelined do
-          it.del "limit_fetch:limit:#{queue}"
-          it.del "limit_fetch:process_limit:#{queue}"
-          it.del "limit_fetch:busy:#{queue}"
-          it.del "limit_fetch:probed:#{queue}"
-          it.del "limit_fetch:pause:#{queue}"
-          it.del "limit_fetch:block:#{queue}"
+        it.pipelined do |pipeline|
+          pipeline.del "limit_fetch:limit:#{queue}"
+          pipeline.del "limit_fetch:process_limit:#{queue}"
+          pipeline.del "limit_fetch:busy:#{queue}"
+          pipeline.del "limit_fetch:probed:#{queue}"
+          pipeline.del "limit_fetch:pause:#{queue}"
+          pipeline.del "limit_fetch:block:#{queue}"
         end
       end
 

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 4.2.0
 platform: ruby
 authors:
+- Dean Perry
 - brainopia
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-13 00:00:00.000000000 Z
+date: 2022-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,13 +18,27 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4'
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4'
-  prerelease: false
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.6.0
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.6.0
 - !ruby/object:Gem::Dependency
   name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
@@ -34,6 +49,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.2
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -42,8 +59,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.2
-  prerelease: false
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -51,13 +80,13 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -65,26 +94,27 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
-description: |2
-      Sidekiq strategy to restrict number of workers
-      which are able to run specified queues simultaneously.
-email: brainopia@evilmartians.com
+description: Sidekiq strategy to restrict number of workers which are able to run
+  specified queues simultaneously.
+email: dean@deanpcmad.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
+- Appraisals
+- CHANGELOG.md
 - Gemfile
-- LICENSE.txt
+- LICENSE
 - README.md
 - Rakefile
 - bench/compare.rb
@@ -100,6 +130,20 @@ files:
 - demo/config/boot.rb
 - demo/config/environment.rb
 - demo/config/environments/development.rb
+- gemfiles/sidekiq_6.0.gemfile
+- gemfiles/sidekiq_6.0.gemfile.lock
+- gemfiles/sidekiq_6.1.gemfile
+- gemfiles/sidekiq_6.1.gemfile.lock
+- gemfiles/sidekiq_6.2.gemfile
+- gemfiles/sidekiq_6.2.gemfile.lock
+- gemfiles/sidekiq_6.3.gemfile
+- gemfiles/sidekiq_6.3.gemfile.lock
+- gemfiles/sidekiq_6.4.gemfile
+- gemfiles/sidekiq_6.4.gemfile.lock
+- gemfiles/sidekiq_6.5.gemfile
+- gemfiles/sidekiq_6.5.gemfile.lock
+- gemfiles/sidekiq_master.gemfile
+- gemfiles/sidekiq_master.gemfile.lock
 - lib/sidekiq-limit_fetch.rb
 - lib/sidekiq/extensions/manager.rb
 - lib/sidekiq/extensions/queue.rb
@@ -117,11 +161,14 @@ files:
 - spec/sidekiq/limit_fetch/semaphore_spec.rb
 - spec/sidekiq/limit_fetch_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/brainopia/sidekiq-limit_fetch
+homepage: https://github.com/deanpcmad/sidekiq-limit_fetch
 licenses:
 - MIT
-metadata: {}
-post_install_message:
+metadata:
+  homepage_uri: https://github.com/deanpcmad/sidekiq-limit_fetch
+  source_code_uri: https://github.com/deanpcmad/sidekiq-limit_fetch
+  changelog_uri: https://github.com/deanpcmad/sidekiq-limit_fetch/blob/master/CHANGELOG.md
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -136,9 +183,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.4
-signing_key:
+rubygems_version: 3.3.7
+signing_key: 
 specification_version: 4
 summary: Sidekiq strategy to support queue limits
 test_files: