sidekiq-limit_fetch 4.3.2 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca583389b5cea0b31200752d2c0dd85a03d38a7ea576c9a299a99367a1c8086d
-  data.tar.gz: 335f49aac1d3ee9128a3c92eb88a25010d906a8b8d8701d1a79b1e41349ca256
+  metadata.gz: 60f78289f95ff030e04baa2f82f05adb60e40a022ca338f06a3c7bffcd69a348
+  data.tar.gz: 237c99bbf64fc49dcb85452bfcbca459ef4745b467455a34c8662e1f523a0663
 SHA512:
-  metadata.gz: 00daee076b583097d361fed04a7128188af6d8a287906a644d0449c01dd8748fd4b8b9d233eddfe1f1da22d0492626088f11e4eb9785dd31da75a47c57437726
-  data.tar.gz: 110b867205d0aea423ec63d8e010d5d1b75398b5aefce782fdb2320fd5e6114b10d041f59306fadb650d3401ea6a8c2fd962a6fd80d793162d8d14061c510e66
+  metadata.gz: bbc1b8489daab83e229590f24a5bd3213804e7fd51b9bb74c1c299812c551e0df06145ae03f9c5f60c993209cd294864d3b044d2b30c27ec138b4ba20fab546c
+  data.tar.gz: 54c0ba14c23a84e8167855c0c759bdb9df315af8aa090bca2d59e930bbfa22011520e7788990c27a5aacb6661b61db7a6a6abdf30afec4de6a5a38fffff60a04

data/.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby: ["2.7", "3.0", "3.1"]
-        appraisal: ['6.0', '6.1', '6.2', '6.3', '6.4', '6.5', 'master']
+        appraisal: ['6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '7.0', 'master']
     services:
       redis:
         image: redis

data/Appraisals

@@ -22,6 +22,10 @@ appraise 'sidekiq-6.5' do
   gem 'sidekiq', '~> 6.5.0'
 end
 
+appraise 'sidekiq-7.0' do
+  gem 'sidekiq', '~> 7.0.0'
+end
+
 appraise 'sidekiq-master' do
   gem 'sidekiq', github: 'mperham/sidekiq'
 end

data/README.md

@@ -1,6 +1,6 @@
 ## Description
 
-*This project has been taken over by [@deanpcmad](https://github.com/deanpcmad)*
+*This project has been taken over by [@deanpcmad](https://github.com/deanpcmad). Original code by [@brainopia](https://github.com/brainopia).*
 
 Sidekiq strategy to support a granular queue control – limiting, pausing, blocking, querying.
 
@@ -150,6 +150,16 @@ To use this mode you need to specify a following line in sidekiq.yml:
 :dynamic: true
 ```
 
+or
+
+```yaml
+  :dynamic:
+    :exclude:
+      - excluded_queue
+```
+
+to exclude `excluded_queue` from dynamic queue
+
 Dynamic queues will be ran at the lowest priority.
 
 ### Maintenance

data/docker-compose.dev.yml

@@ -0,0 +1,13 @@
+version: "3.8"
+services: 
+  redis:
+    container_name: redis
+    image: "redis:6.2-alpine"
+    ports:
+      - 6379:6379
+    volumes:
+      - redis-data:/var/lib/redis
+
+volumes:
+  redis-data:
+    driver: local

data/gemfiles/sidekiq_6.0.gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: ..
   specs:
-    sidekiq-limit_fetch (4.3.2)
-      redis (>= 4.6.0)
-      sidekiq (>= 4)
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
 
 GEM
   remote: https://rubygems.org/
@@ -53,4 +52,4 @@ DEPENDENCIES
   sidekiq-limit_fetch!
 
 BUNDLED WITH
-   2.3.21
+   2.3.5

data/gemfiles/sidekiq_6.1.gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: ..
   specs:
-    sidekiq-limit_fetch (4.3.2)
-      redis (>= 4.6.0)
-      sidekiq (>= 4)
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
 
 GEM
   remote: https://rubygems.org/
@@ -50,4 +49,4 @@ DEPENDENCIES
   sidekiq-limit_fetch!
 
 BUNDLED WITH
-   2.3.21
+   2.3.5

data/gemfiles/sidekiq_6.2.gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: ..
   specs:
-    sidekiq-limit_fetch (4.3.2)
-      redis (>= 4.6.0)
-      sidekiq (>= 4)
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
 
 GEM
   remote: https://rubygems.org/
@@ -50,4 +49,4 @@ DEPENDENCIES
   sidekiq-limit_fetch!
 
 BUNDLED WITH
-   2.3.21
+   2.3.5

data/gemfiles/sidekiq_6.3.gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: ..
   specs:
-    sidekiq-limit_fetch (4.3.2)
-      redis (>= 4.6.0)
-      sidekiq (>= 4)
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
 
 GEM
   remote: https://rubygems.org/
@@ -50,4 +49,4 @@ DEPENDENCIES
   sidekiq-limit_fetch!
 
 BUNDLED WITH
-   2.3.21
+   2.3.5

data/gemfiles/sidekiq_6.4.gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: ..
   specs:
-    sidekiq-limit_fetch (4.3.2)
-      redis (>= 4.6.0)
-      sidekiq (>= 4)
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
 
 GEM
   remote: https://rubygems.org/
@@ -50,4 +49,4 @@ DEPENDENCIES
   sidekiq-limit_fetch!
 
 BUNDLED WITH
-   2.3.21
+   2.3.5

data/gemfiles/sidekiq_6.5.gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: ..
   specs:
-    sidekiq-limit_fetch (4.3.2)
-      redis (>= 4.6.0)
-      sidekiq (>= 4)
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
 
 GEM
   remote: https://rubygems.org/
@@ -50,4 +49,4 @@ DEPENDENCIES
   sidekiq-limit_fetch!
 
 BUNDLED WITH
-   2.3.21
+   2.3.5

data/gemfiles/sidekiq_7.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 7.0.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_7.0.gemfile.lock

@@ -0,0 +1,58 @@
+PATH
+  remote: ..
+  specs:
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    concurrent-ruby (1.1.10)
+    connection_pool (2.3.0)
+    diff-lcs (1.5.0)
+    rack (3.0.2)
+    rake (13.0.6)
+    redis (5.0.5)
+      redis-client (>= 0.9.0)
+    redis-client (0.11.2)
+      connection_pool
+    redis-namespace (1.10.0)
+      redis (>= 4)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    sidekiq (7.0.2)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      rack (>= 2.2.4)
+      redis-client (>= 0.11.0)
+    thor (1.2.1)
+
+PLATFORMS
+  x86_64-darwin-21
+  x86_64-linux
+
+DEPENDENCIES
+  appraisal
+  rake
+  redis-namespace (~> 1.5, >= 1.5.2)
+  rspec
+  sidekiq (~> 7.0.0)
+  sidekiq-limit_fetch!
+
+BUNDLED WITH
+   2.3.5

data/gemfiles/sidekiq_master.gemfile.lock

@@ -10,9 +10,8 @@ GIT
 PATH
   remote: ..
   specs:
-    sidekiq-limit_fetch (4.3.2)
-      redis (>= 4.6.0)
-      sidekiq (>= 4)
+    sidekiq-limit_fetch (4.4.0)
+      sidekiq (>= 6)
 
 GEM
   remote: https://rubygems.org/
@@ -55,4 +54,4 @@ DEPENDENCIES
   sidekiq-limit_fetch!
 
 BUNDLED WITH
-   2.3.21
+   2.3.5

data/lib/sidekiq/extensions/manager.rb

@@ -1,7 +1,12 @@
 class Sidekiq::Manager
   module InitLimitFetch
     def initialize(options={})
-      options[:fetch] = Sidekiq::LimitFetch
+      if Sidekiq::LimitFetch.post_7?
+        options.config[:fetch] = Sidekiq::LimitFetch
+      else
+        options[:fetch] = Sidekiq::LimitFetch
+      end
+      
       super
     end
 

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -27,7 +27,7 @@ module Sidekiq::LimitFetch::Global
 
     def old_processes
       all_processes.reject do |process|
-        Sidekiq.redis {|it| it.get heartbeat_key process }
+        Sidekiq.redis {|it| it.get heartbeat_key process } == '1'
       end
     end
 
@@ -39,7 +39,12 @@ module Sidekiq::LimitFetch::Global
 
     def handle_dynamic_queues
       queues = Sidekiq::LimitFetch::Queues
-      queues.handle Sidekiq::Queue.all.map(&:name) if queues.dynamic?
+      return unless queues.dynamic?
+
+      available_queues = Sidekiq::Queue.all.map(&:name).reject do |it|
+        queues.dynamic_exclude.include? it
+      end
+      queues.handle available_queues
     end
 
     private
@@ -47,7 +52,7 @@ module Sidekiq::LimitFetch::Global
     def update_heartbeat(ttl)
       Sidekiq.redis do |it|
         it.multi do |pipeline|
-          pipeline.set heartbeat_key, true
+          pipeline.set heartbeat_key, '1'
           pipeline.sadd PROCESS_SET, [Selector.uuid]
           pipeline.expire heartbeat_key, ttl
         end

data/lib/sidekiq/limit_fetch/global/selector.rb

@@ -25,20 +25,24 @@ module Sidekiq::LimitFetch::Global
     def redis_eval(script_name, args)
       Sidekiq.redis do |it|
         begin
-          it.evalsha send("redis_#{script_name}_sha"), argv: args
-        rescue Redis::CommandError => error
+          it.evalsha send("redis_#{script_name}_sha"), [], args
+        rescue Sidekiq::LimitFetch::RedisCommandError => error
           raise unless error.message.include? 'NOSCRIPT'
-          it.eval send("redis_#{script_name}_script"), argv: args
+          if Sidekiq::LimitFetch.post_7?
+            it.eval send("redis_#{script_name}_script"), 0, *args
+          else
+            it.eval send("redis_#{script_name}_script"), argv: args
+          end
         end
       end
     end
 
     def redis_acquire_sha
-      @acquire_sha ||= Digest::SHA1.hexdigest redis_acquire_script
+      @acquire_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_acquire_script
     end
 
     def redis_release_sha
-      @release_sha ||= Digest::SHA1.hexdigest redis_release_script
+      @release_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_release_script
     end
 
     def redis_acquire_script

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -77,11 +77,11 @@ module Sidekiq::LimitFetch::Global
     end
 
     def pause
-      redis {|it| it.set "#{PREFIX}:pause:#@name", true }
+      redis {|it| it.set "#{PREFIX}:pause:#@name", '1' }
     end
 
     def pause_for_ms ms
-      redis {|it| it.psetex "#{PREFIX}:pause:#@name", ms, true }
+      redis {|it| it.psetex "#{PREFIX}:pause:#@name", ms, 1 }
     end
 
     def unpause
@@ -89,11 +89,11 @@ module Sidekiq::LimitFetch::Global
     end
 
     def paused?
-      redis {|it| it.get "#{PREFIX}:pause:#@name" }
+      redis {|it| it.get "#{PREFIX}:pause:#@name" } == '1'
     end
 
     def block
-      redis {|it| it.set "#{PREFIX}:block:#@name", true }
+      redis {|it| it.set "#{PREFIX}:block:#@name", '1' }
     end
 
     def block_except(*queues)
@@ -106,7 +106,7 @@ module Sidekiq::LimitFetch::Global
     end
 
     def blocking?
-      redis {|it| it.get "#{PREFIX}:block:#@name" }
+      redis {|it| it.get "#{PREFIX}:block:#@name" } == '1'
     end
 
     def clear_limits

data/lib/sidekiq/limit_fetch/queues.rb

@@ -4,9 +4,20 @@ module Sidekiq::LimitFetch::Queues
   THREAD_KEY = :acquired_queues
 
   def start(options)
+    if Sidekiq::LimitFetch.post_7?
+      options = options.config
+    end
+
     @queues         = options[:queues]
     @startup_queues = options[:queues].dup
-    @dynamic        = options[:dynamic]
+
+    if options[:dynamic].is_a? Hash
+      @dynamic         = true
+      @dynamic_exclude = options[:dynamic][:exclude] || []
+    else
+      @dynamic = options[:dynamic]
+      @dynamic_exclude = []
+    end
 
     @limits         = options[:limits] || {}
     @process_limits = options[:process_limits] || {}
@@ -44,6 +55,10 @@ module Sidekiq::LimitFetch::Queues
     @startup_queues.include?(queue)
   end
 
+  def dynamic_exclude
+    @dynamic_exclude
+  end
+
   def add(queues)
     return unless queues
     queues.each do |queue|

data/lib/sidekiq/limit_fetch.rb

@@ -18,6 +18,13 @@ module Sidekiq::LimitFetch
 
   extend self
 
+  def post_7?
+    @post_7 ||= Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('7.0.0')
+  end
+
+  RedisBaseConnectionError = post_7? ? RedisClient::ConnectionError : Redis::BaseConnectionError
+  RedisCommandError = post_7? ? RedisClient::CommandError : Redis::CommandError
+
   def new(_)
     self
   end
@@ -45,10 +52,10 @@ module Sidekiq::LimitFetch
 
   def redis_retryable
     yield
-  rescue Redis::BaseConnectionError
+  rescue RedisBaseConnectionError
     sleep TIMEOUT
     retry
-  rescue Redis::CommandError => error
+  rescue RedisCommandError => error
     # If Redis was restarted and is still loading its snapshot,
     # then we should treat this as a temporary connection error too.
     if error.message =~ /^LOADING/
@@ -70,7 +77,15 @@ module Sidekiq::LimitFetch
       sleep TIMEOUT  # there are no queues to handle, so lets sleep
       []             # and return nothing
     else
-      redis_retryable { Sidekiq.redis { |it| it.brpop *queues, timeout: TIMEOUT } }
+      redis_retryable do
+        Sidekiq.redis do |it|
+          if post_7?
+            it.blocking_call(false, "brpop", *queues, TIMEOUT)
+          else
+            it.brpop(*queues, timeout: TIMEOUT)
+          end
+        end
+      end
     end
   end
 end

data/sidekiq-limit_fetch.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '4.3.2'
+  gem.version       = '4.4.0'
   gem.license       = 'MIT'
-  gem.authors       = ['Dean Perry', 'brainopia']
+  gem.authors       = ['Dean Perry']
   gem.email         = 'dean@deanpcmad.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'
   gem.homepage      = 'https://github.com/deanpcmad/sidekiq-limit_fetch'
@@ -16,8 +16,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep %r{^spec/}
   gem.require_paths = %w(lib)
 
-  gem.add_dependency 'sidekiq', '>= 4'
-  gem.add_dependency 'redis', '>= 4.6.0'
+  gem.add_dependency 'sidekiq', '>= 6'
   gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2'
   gem.add_development_dependency 'appraisal'
   gem.add_development_dependency 'rspec'

data/spec/sidekiq/limit_fetch/global/monitor_spec.rb

@@ -3,12 +3,12 @@ RSpec.describe Sidekiq::LimitFetch::Global::Monitor do
   let(:ttl) { 1 }
   let(:queue) { Sidekiq::Queue[name] }
   let(:name) { 'default' }
+  let(:timeout) { 0.5 }
 
-  before { monitor }
   after { monitor.kill }
 
   context 'old locks' do
-    let(:timeout) { 0.5 }
+    before { monitor }
 
     it 'should remove invalidated old locks' do
       2.times { queue.acquire }
@@ -29,5 +29,59 @@ RSpec.describe Sidekiq::LimitFetch::Global::Monitor do
       sleep 2*ttl
       expect(queue.probed).to eq 0
     end
+
+  end
+
+  context 'dynamic queue' do
+    let(:limits) do
+      {
+        'queue1' => 3,
+        'queue2' => 3,
+      }
+    end
+    let(:queues) { %w[queue1 queue2] }
+    let(:queue) { Sidekiq::LimitFetch::Queues }
+    let(:options) do
+      {
+        limits: limits,
+        queues: queues,
+      }
+    end
+
+    it 'should add dynamic queue' do
+      queue.start(options.merge({ dynamic: true }))
+      monitor
+
+      expect(queue.instance_variable_get(:@queues)).not_to include('queue3')
+
+      Sidekiq.redis do |it|
+        it.sadd 'queues', 'queue3'
+      end
+
+      sleep 2*ttl
+      expect(queue.instance_variable_get(:@queues)).to include('queue3')
+
+      Sidekiq.redis do |it|
+        it.srem 'queues', 'queue3'
+      end
+    end
+
+    it 'should exclude excluded dynamic queue' do
+      queue.start(options.merge({ dynamic: { exclude: ['queue4'] } }))
+      monitor
+
+      expect(queue.instance_variable_get(:@queues)).not_to include('queue4')
+
+      Sidekiq.redis do |it|
+        it.sadd 'queues', 'queue4'
+      end
+
+      sleep 2*ttl
+      expect(queue.instance_variable_get(:@queues)).not_to include('queue4')
+
+      Sidekiq.redis do |it|
+        it.srem 'queues', 'queue4'
+      end
+    end
   end
 end

data/spec/spec_helper.rb

@@ -1,7 +1,13 @@
 require 'sidekiq/limit_fetch'
 
-Sidekiq.logger = nil
-Sidekiq.redis = { namespace: ENV['namespace'] }
+if Sidekiq::LimitFetch.post_7?
+  Sidekiq.configure_embed do |config|
+    config.logger = nil
+  end
+else
+  Sidekiq.logger = nil
+  Sidekiq.redis = { namespace: ENV['namespace'] }
+end
 
 RSpec.configure do |config|
   config.order = :random

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: 4.3.2
+  version: 4.4.0
 platform: ruby
 authors:
 - Dean Perry
-- brainopia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-01 00:00:00.000000000 Z
+date: 2023-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,28 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4'
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.6.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.6.0
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
@@ -130,6 +115,7 @@ files:
 - demo/config/boot.rb
 - demo/config/environment.rb
 - demo/config/environments/development.rb
+- docker-compose.dev.yml
 - gemfiles/sidekiq_6.0.gemfile
 - gemfiles/sidekiq_6.0.gemfile.lock
 - gemfiles/sidekiq_6.1.gemfile
@@ -142,6 +128,8 @@ files:
 - gemfiles/sidekiq_6.4.gemfile.lock
 - gemfiles/sidekiq_6.5.gemfile
 - gemfiles/sidekiq_6.5.gemfile.lock
+- gemfiles/sidekiq_7.0.gemfile
+- gemfiles/sidekiq_7.0.gemfile.lock
 - gemfiles/sidekiq_master.gemfile
 - gemfiles/sidekiq_master.gemfile.lock
 - lib/sidekiq-limit_fetch.rb