rcgt-sidekiq-limit_fetch 3.4.1

data/spec/sidekiq/limit_fetch/queues_spec.rb

@@ -0,0 +1,100 @@
+RSpec.describe Sidekiq::LimitFetch::Queues do
+  let(:queues)         { %w[queue1 queue2] }
+  let(:limits)         {{ 'queue1' => 3 }}
+  let(:strict)         { true }
+  let(:blocking)       {}
+  let(:process_limits) {{ 'queue2' => 3 }}
+
+  let(:options) do
+    { queues:   queues,
+      limits:   limits,
+      strict:   strict,
+      blocking: blocking,
+      process_limits: process_limits }
+  end
+
+  before { subject.start options }
+
+  it 'should acquire queues' do
+    subject.acquire
+    expect(Sidekiq::Queue['queue1'].probed).to eq 1
+    expect(Sidekiq::Queue['queue2'].probed).to eq 1
+  end
+
+  it 'should acquire dynamically blocking queues' do
+    subject.acquire
+    expect(Sidekiq::Queue['queue1'].probed).to eq 1
+    expect(Sidekiq::Queue['queue2'].probed).to eq 1
+
+    Sidekiq::Queue['queue1'].block
+
+    subject.acquire
+    expect(Sidekiq::Queue['queue1'].probed).to eq 2
+    expect(Sidekiq::Queue['queue2'].probed).to eq 1
+  end
+
+  it 'should block except given queues' do
+    Sidekiq::Queue['queue1'].block_except 'queue2'
+    subject.acquire
+    expect(Sidekiq::Queue['queue1'].probed).to eq 1
+    expect(Sidekiq::Queue['queue2'].probed).to eq 1
+
+    Sidekiq::Queue['queue1'].block_except 'queue404'
+    subject.acquire
+    expect(Sidekiq::Queue['queue1'].probed).to eq 2
+    expect(Sidekiq::Queue['queue2'].probed).to eq 1
+  end
+
+  it 'should release queues' do
+    subject.acquire
+    subject.release_except nil
+    expect(Sidekiq::Queue['queue1'].probed).to eq 0
+    expect(Sidekiq::Queue['queue2'].probed).to eq 0
+  end
+
+  it 'should release queues except selected' do
+    subject.acquire
+    subject.release_except 'queue:queue1'
+    expect(Sidekiq::Queue['queue1'].probed).to eq 1
+    expect(Sidekiq::Queue['queue2'].probed).to eq 0
+  end
+
+  it 'should release when no queues was acquired' do
+    queues.each {|name| Sidekiq::Queue[name].pause }
+    subject.acquire
+    expect { subject.release_except nil }.not_to raise_exception
+  end
+
+  context 'blocking' do
+    let(:blocking) { %w(queue1) }
+
+    it 'should acquire blocking queues' do
+      3.times { subject.acquire }
+      expect(Sidekiq::Queue['queue1'].probed).to eq 3
+      expect(Sidekiq::Queue['queue2'].probed).to eq 1
+    end
+  end
+
+  it 'should set limits' do
+    subject
+    expect(Sidekiq::Queue['queue1'].limit).to eq 3
+    expect(Sidekiq::Queue['queue2'].limit).not_to be
+  end
+
+  it 'should set process_limits' do
+    subject
+    expect(Sidekiq::Queue['queue2'].process_limit).to eq 3
+  end
+
+  context 'without strict flag' do
+    let(:strict) { false }
+
+    it 'should retrieve weighted queues' do
+      expect(subject.ordered_queues).to match_array(%w(queue1 queue2))
+    end
+  end
+
+  it 'with strict flag should retrieve strictly ordered queues' do
+    expect(subject.ordered_queues).to eq %w(queue1 queue2)
+  end
+end

data/spec/sidekiq/limit_fetch/semaphore_spec.rb

@@ -0,0 +1,63 @@
+RSpec.describe 'semaphore' do
+  let(:name) { 'default' }
+  subject { Sidekiq::LimitFetch::Global::Semaphore.new name }
+
+  it 'should have no limit by default' do
+    expect(subject.limit).not_to be
+  end
+
+  it 'should set limit' do
+    subject.limit = 4
+    expect(subject.limit).to eq 4
+  end
+
+  it 'should acquire and count active tasks' do
+    3.times { subject.acquire }
+    expect(subject.probed).to eq 3
+  end
+
+  it 'should acquire tasks with regard to limit' do
+    subject.limit = 4
+    6.times { subject.acquire }
+    expect(subject.probed).to eq 4
+  end
+
+  it 'should acquire tasks with regard to process limit' do
+    subject.process_limit = 4
+    6.times { subject.acquire }
+    expect(subject.probed).to eq 4
+  end
+
+  it 'should release active tasks' do
+    6.times { subject.acquire }
+    3.times { subject.release }
+    expect(subject.probed).to eq 3
+  end
+
+  it 'should pause tasks' do
+    3.times { subject.acquire }
+    subject.pause
+    2.times { subject.acquire }
+    expect(subject.probed).to eq 3
+    2.times { subject.release }
+    expect(subject.probed).to eq 1
+  end
+
+  it 'should unpause tasks' do
+    subject.pause
+    3.times { subject.acquire }
+    subject.unpause
+    2.times { subject.acquire }
+    expect(subject.probed).to eq 2
+  end
+
+  it 'should pause tasks for a limited time' do
+    3.times { subject.acquire }
+    subject.pause_for_ms 50
+    2.times { subject.acquire }
+    expect(subject.probed).to eq 3
+    sleep(100.0 / 1000)
+    2.times { subject.acquire }
+    expect(subject.probed).to eq 5
+  end
+end

data/spec/sidekiq/limit_fetch_spec.rb

@@ -0,0 +1,48 @@
+Thread.abort_on_exception = true
+
+RSpec.describe Sidekiq::LimitFetch do
+  let(:options) {{ queues: queues, limits: limits }}
+  let(:queues) { %w(queue1 queue1 queue2 queue2) }
+  let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
+
+  before do
+    subject::Queues.start options 
+
+    Sidekiq.redis do |it|
+      it.del 'queue:queue1'
+      it.lpush 'queue:queue1', 'task1'
+      it.lpush 'queue:queue1', 'task2'
+      it.expire 'queue:queue1', 30
+    end
+  end
+
+  it 'should acquire lock on queue for execution' do
+    work = subject.retrieve_work
+    expect(work.queue_name).to eq 'queue1'
+    expect(work.job).to eq 'task1'
+
+    expect(Sidekiq::Queue['queue1'].busy).to eq 1
+    expect(Sidekiq::Queue['queue2'].busy).to eq 0
+
+    expect(subject.retrieve_work).not_to be
+    work.requeue
+
+    expect(Sidekiq::Queue['queue1'].busy).to eq 0
+    expect(Sidekiq::Queue['queue2'].busy).to eq 0
+
+    work = subject.retrieve_work
+    expect(work.job).to eq 'task1'
+
+    expect(Sidekiq::Queue['queue1'].busy).to eq 1
+    expect(Sidekiq::Queue['queue2'].busy).to eq 0
+
+    expect(subject.retrieve_work).not_to be
+    work.acknowledge
+
+    expect(Sidekiq::Queue['queue1'].busy).to eq 0
+    expect(Sidekiq::Queue['queue2'].busy).to eq 0
+
+    work = subject.retrieve_work
+    expect(work.job).to eq 'task2'
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+require 'sidekiq/limit_fetch'
+
+Sidekiq.logger = nil
+Sidekiq.redis = { namespace: ENV['namespace'] }
+
+RSpec.configure do |config|
+  config.order = :random
+  config.disable_monkey_patching!
+  config.raise_errors_for_deprecations!
+  config.before do
+    Sidekiq::Queue.reset_instances!
+    Sidekiq.redis do |it|
+      clean_redis = ->(queue) do
+        it.pipelined do
+          it.del "limit_fetch:limit:#{queue}"
+          it.del "limit_fetch:process_limit:#{queue}"
+          it.del "limit_fetch:busy:#{queue}"
+          it.del "limit_fetch:probed:#{queue}"
+          it.del "limit_fetch:pause:#{queue}"
+          it.del "limit_fetch:block:#{queue}"
+        end
+      end
+
+      clean_redis.call(name) if defined?(name)
+      queues.each(&clean_redis) if defined?(queues) and queues.is_a? Array
+    end
+  end
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: rcgt-sidekiq-limit_fetch
+version: !ruby/object:Gem::Version
+  version: 3.4.1
+platform: ruby
+authors:
+- RCGT Consulting Inc.
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-02-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: "    A fork of https://github.com/brainopia/sidekiq-limit_fetch\n    \n
+  \   Sidekiq strategy to restrict number of workers\n    which are able to run specified
+  queues simultaneously.\n"
+email: kumanan.yogaratnam@rcgtconsulting.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bench/compare.rb
+- demo/Gemfile
+- demo/README.md
+- demo/Rakefile
+- demo/app/workers/a_worker.rb
+- demo/app/workers/b_worker.rb
+- demo/app/workers/c_worker.rb
+- demo/app/workers/fast_worker.rb
+- demo/app/workers/slow_worker.rb
+- demo/config/application.rb
+- demo/config/boot.rb
+- demo/config/environment.rb
+- demo/config/environments/development.rb
+- lib/sidekiq-limit_fetch.rb
+- lib/sidekiq/extensions/manager.rb
+- lib/sidekiq/extensions/queue.rb
+- lib/sidekiq/limit_fetch.rb
+- lib/sidekiq/limit_fetch/global/monitor.rb
+- lib/sidekiq/limit_fetch/global/selector.rb
+- lib/sidekiq/limit_fetch/global/semaphore.rb
+- lib/sidekiq/limit_fetch/instances.rb
+- lib/sidekiq/limit_fetch/queues.rb
+- lib/sidekiq/limit_fetch/unit_of_work.rb
+- rcgt-sidekiq-limit_fetch.gemspec
+- spec/sidekiq/extensions/queue_spec.rb
+- spec/sidekiq/limit_fetch/global/monitor_spec.rb
+- spec/sidekiq/limit_fetch/queues_spec.rb
+- spec/sidekiq/limit_fetch/semaphore_spec.rb
+- spec/sidekiq/limit_fetch_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/RCGTConsulting/sidekiq-limit_fetch
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Sidekiq strategy to support queue limits
+test_files:
+- spec/sidekiq/extensions/queue_spec.rb
+- spec/sidekiq/limit_fetch/global/monitor_spec.rb
+- spec/sidekiq/limit_fetch/queues_spec.rb
+- spec/sidekiq/limit_fetch/semaphore_spec.rb
+- spec/sidekiq/limit_fetch_spec.rb
+- spec/spec_helper.rb