HornsAndHooves-sidekiq-limit_fetch 4.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 87f74cf9c2b4a225260785a8793e7616f99f4e49215517b6edbc352fd055a9a9
+  data.tar.gz: '0439c9cd2c4950cc826e67245e5cb7a7c377d49158a8a4cdc16d4e0a08d497d6'
+SHA512:
+  metadata.gz: 25bbb48cfa97773220546d0fe1bd0dce302b1b5f155fa2b36949ea8bb8fdd3204ece5d8ef107b3abf0164a6fcadc169119b959ed67a80552a1a6b4d1e5c3af29
+  data.tar.gz: 10e9df0a5823b98eeb684ba0ed4787b057b6bd2a7b62780f6fc288419bf6ac5ab67d678b8c7b1a911cd318f8ef3058aad4c12e6d3f35627f4d4269e87f80456b

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+pkg/
+.bundle/
+coverage/

data/.rspec

@@ -0,0 +1,5 @@
+--color
+--profile
+--backtrace
+--require spec_helper
+--format documentation

data/.rubocop.yml

@@ -0,0 +1,34 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.7
+  Exclude:
+    - 'gemfiles/*'
+
+Style/Documentation:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*.rb'
+    - 'demo/Rakefile'
+
+Metrics/ModuleLength:
+  Exclude:
+    - 'lib/sidekiq/limit_fetch/queues.rb'
+
+Metrics/ClassLength:
+  Exclude:
+    - 'lib/sidekiq/limit_fetch/global/semaphore.rb'
+
+Naming/VariableNumber:
+  EnforcedStyle: snake_case
+
+Naming/FileName:
+  Exclude:
+    - lib/sidekiq-limit_fetch.rb
+
+Gemspec/DevelopmentDependencies:
+  EnforcedStyle: gemspec
+  Exclude:
+    - 'demo/*'

data/.ruby-gemset

@@ -0,0 +1 @@
+sidekiq-limit_fetch

data/.ruby-version

@@ -0,0 +1 @@
+3.3.9

data/CHANGELOG.md

@@ -0,0 +1,37 @@
+# Changelog
+
+## [4.5.0] - 2026-06-15
+
+This project was taken over by [@HornsAndHooves](https://github.com/HornsAndHooves)
+
+- Remove compatibility with Sidekiq < 8
+- Fix issue with queue weights specified via CLI not processing jobs
+
+## [4.3.2] - 2022-09-01
+
+- #139 - Fix Redis deprecation warnings from [@adamzapasnik](https://github.com/adamzapasnik)
+
+## [4.3.1] - 2022-08-23
+
+- #137 - Fix deprecation of passing timeout as positional argument to brpop from [@cgunther](https://github.com/cgunther)
+
+## [4.3.0] - 2022-08-16
+
+- #135 - Some extra fixes for Sidekiq 6.5 (fixes #128, #130, #131) from [@BobbyMcWho](https://github.com/BobbyMcWho)
+
+## [4.2.0] - 2022-06-09
+
+- #127 - Fix for Sidekiq 6.5 internal change vias PR #128 from [@evgeniradev][https://github.com/evgeniradev]
+- testing changes: stop supporting Sidekiq < 6, add tests for Sidekiq 6.5, stop testing on ruby 2.6 EOL
+
+## [4.1.0] - 2022-03-29
+
+- #101 - Fix stuck queues bug on Redis restart from [@907th](https://github.com/907th).
+
+## [4.0.0] - 2022-03-26
+
+This project was taken over by [@deanpcmad](https://github.com/deanpcmad)
+
+- #120 - Migrate CI to GitHub Actions from [@petergoldstein](https://github.com/petergoldstein).
+- #124 - Fixed redis v4.6.0 pipelines deprecation warning from [@iurev](https://github.com/iurev).
+- #83  - Processing dynamic queues from [@alexey-yanchenko](https://github.com/alexey-yanchenko).

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2022 Dean Perry
+Copyright (c) 2013 brainopia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,165 @@
+## Description
+
+*This is a fork of <https://github.com/deanpcmad/sidekiq-limit_fetch>. This version does not support Sidekiq < 8, if you are using a Sidekiq version below v8 you must use the original gem.*
+
+Sidekiq strategy to support a granular queue control – limiting, pausing, blocking, querying.
+
+### Installation
+
+Add this line to your application's Gemfile:
+
+```
+gem 'HornsAndHooves-sidekiq-limit_fetch'
+```
+
+Then `bundle install`.
+
+### Limitations
+
+> [!WARNING]
+> At this moment, `sidekiq-limit_fetch` is incompatible with
+> Sidekiq Pro's [super_fetch](https://github.com/sidekiq/sidekiq/wiki/Reliability#using-super_fetch),
+> [sidekiq-rate-limiter](https://github.com/enova/sidekiq-rate-limiter),
+> and any other plugin that rewrites fetch strategy of Sidekiq.
+
+### Usage
+
+If you are using this with Rails, you don't need to require it as it's done automatically.
+
+To use this Gem in other Ruby projects, just add `require 'HornsAndHooves-sidekiq-limit_fetch'`.
+
+### Limits
+
+Specify limits which you want to place on queues inside `sidekiq.yml`:
+
+```yaml
+:limits:
+  queue_name1: 5
+  queue_name2: 10
+```
+
+Or set it dynamically in your code:
+```ruby
+Sidekiq::Queue['queue_name1'].limit = 5
+Sidekiq::Queue['queue_name2'].limit = 10
+```
+
+In these examples, tasks for the `queue_name1` will be run by at most 5
+workers at the same time and the `queue_name2` will have no more than 10
+workers simultaneously.
+
+Ability to set limits dynamically allows you to resize worker
+distribution among queues any time you want.
+
+### Limits per process
+
+If you use multiple Sidekiq processes then you can specify limits per process:
+
+```yaml
+:process_limits:
+  queue_name: 2
+```
+
+Or set it in your code:
+
+```ruby
+Sidekiq::Queue['queue_name'].process_limit = 2
+```
+
+### Busy workers by queue
+
+You can see how many workers currently handling a queue:
+
+```ruby
+Sidekiq::Queue['name'].busy # number of busy workers
+```
+
+### Pauses
+
+You can also pause your queues temporarily. Upon continuing their limits
+will be preserved.
+
+```ruby
+Sidekiq::Queue['name'].pause # prevents workers from running tasks from this queue
+Sidekiq::Queue['name'].paused? # => true
+Sidekiq::Queue['name'].unpause # allows workers to use the queue
+Sidekiq::Queue['name'].pause_for_ms(1000) # will pause for a second
+```
+
+### Blocking queue mode
+
+If you use strict queue ordering (it will be used if you don't specify queue weights)
+then you can set blocking status for queues. It means if a blocking
+queue task is executing then no new task from lesser priority queues will
+be ran. Eg,
+
+```yaml
+:queues:
+  - a
+  - b
+  - c
+:blocking:
+  - b
+```
+
+In this case when a task for `b` queue is ran no new task from `c` queue
+will be started.
+
+You can also enable and disable blocking mode for queues on the fly:
+
+```ruby
+Sidekiq::Queue['name'].block
+Sidekiq::Queue['name'].blocking? # => true
+Sidekiq::Queue['name'].unblock
+```
+
+### Advanced blocking queues
+
+You can also block on array of queues. It means when any of them is
+running only queues higher and queues from their blocking group can
+run. It will be easier to understand with an example:
+
+```yaml
+:queues:
+  - a
+  - b
+  - c
+  - d
+:blocking:
+  - [b, c]
+```
+
+In this case tasks from `d` will be blocked when a task from queue `b` or `c` is executed.
+
+You can dynamically set exceptions for queue blocking:
+
+```ruby
+Sidekiq::Queue['queue1'].block_except 'queue2'
+```
+
+### Dynamic queues
+
+You can support dynamic queues (that are not listed in `sidekiq.yml` but
+that have tasks pushed to them (usually with `Sidekiq::Client.push`)).
+
+To use this mode you need to specify a following line in `sidekiq.yml`:
+
+```yaml
+:dynamic: true
+```
+
+or
+
+```yaml
+  :dynamic:
+    :exclude:
+      - excluded_queue
+```
+
+to exclude `excluded_queue` from dynamic queue
+
+Dynamic queues will be ran at the lowest priority.
+
+### Maintenance
+
+If you use `flushdb`, restart the Sidekiq process to re-populate the dynamic configuration.

data/Rakefile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new
+
+task :default do
+  rspec = Rake::Task[:spec]
+  rspec.invoke
+  ENV['namespace'] = 'namespace'
+  rspec.reenable
+  rspec.invoke
+end

data/bench/compare.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+require 'sidekiq/cli'
+require 'sidekiq/api'
+
+total       = (ARGV.shift || 50).to_i
+concurrency = ARGV.shift || 1
+limit       = ARGV.shift
+
+if limit
+  limit = nil if limit == 'nil'
+
+  $LOAD_PATH.unshift File.expand_path '../lib'
+  require 'sidekiq-limit_fetch'
+  Sidekiq::Queue['inline'].limit = limit
+  Sidekiq.redis { |it| it.del 'limit_fetch:probed:inline' }
+  Sidekiq::LimitFetch::Queues.send(:define_method, :set) { |*| } # rubocop:disable Lint/EmptyBlock
+end
+
+Sidekiq::Queue.new('inline').clear
+
+class FastJob
+  include Sidekiq::Worker
+  sidekiq_options queue: :inline
+
+  def perform(index)
+    puts "job N#{index} is finished"
+  end
+end
+
+class FinishJob
+  include Sidekiq::Worker
+  sidekiq_options queue: :inline
+
+  def perform
+    Process.kill 'INT', 0
+  end
+end
+
+total.times { |i| FastJob.perform_async i + 1 }
+FinishJob.perform_async
+
+Sidekiq::CLI.instance.tap do |cli|
+  %w[validate! boot_system].each { |stub| cli.define_singleton_method(stub) {} } # rubocop:disable Lint/EmptyBlock
+  cli.parse ['-q inline', '-q other', "-c #{concurrency}"]
+
+  # rubocop:disable Lint/RescueException
+  # rubocop:disable Lint/SuppressedException
+  puts Benchmark.measure do
+    cli.run
+  rescue Exception
+  end
+  # rubocop:enable Lint/SuppressedException
+  # rubocop:enable Lint/RescueException
+end

data/demo/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'launchy'
+gem 'rails'
+gem 'sidekiq', github: 'mperham/sidekiq'
+gem 'sidekiq-limit_fetch', path: '..'

data/demo/README.md

@@ -0,0 +1,37 @@
+This is a demo rails app with a configured sidekiq-limit_fetch.
+
+Its purpose is to check whether plugin works in certain situations.
+
+Application is preconfigured with two workers:
+- `app/workers/fast_worker.rb` which does `sleep 0.2`
+- `app/workers/slow_worker.rb` which does `sleep 1`
+
+There is also a rake task which can be invoked as `bundle exec rake demo:limit`:
+
+- it prefills sidekiq tasks
+
+```ruby
+  100.times do
+    SlowWorker.perform_async
+    FastWorker.perform_async
+  end
+```
+- sets sidekiq config
+
+```yaml
+  :verbose: false
+  :concurrency: 4
+  :queues:
+    - slow
+    - fast
+  :limits:
+    slow: 1
+```
+
+- and launches a sidekiq admin page with overview of queues in browser.
+The page is set to live-poll so effects of limits can be seen directly.
+
+
+To change simulation modify `Rakefile` or workers.
+
+Any bugs related to the plugin should be demonstrated with a reproduction from this base app.

data/demo/Rakefile

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require File.expand_path('config/application', __dir__)
+Demo::Application.load_tasks
+
+namespace :demo do
+  task limit: :environment do
+    puts '=> Creating sidekiq tasks'
+
+    100.times do
+      SlowWorker.perform_async
+      FastWorker.perform_async
+    end
+
+    run_sidekiq_monitoring
+    run_sidekiq_workers config: <<-YAML
+      :verbose: false
+      :concurrency: 4
+      :queues:
+        - slow
+        - fast
+      :limits:
+        slow: 1
+    YAML
+  end
+
+  task blocking: :environment do
+    puts '=> Creating sidekiq tasks'
+
+    AWorker.perform_async
+    BWorker.perform_async
+    CWorker.perform_async
+
+    run_sidekiq_monitoring
+    run_sidekiq_workers config: <<-YAML
+      :verbose: false
+      :concurrency: 4
+      :queues:
+        - a
+        - b
+        - c
+      :blocking:
+        - a
+    YAML
+  end
+
+  task advanced_blocking: :environment do
+    puts '=> Creating sidekiq tasks'
+
+    AWorker.perform_async
+    BWorker.perform_async
+    CWorker.perform_async
+
+    run_sidekiq_monitoring
+    run_sidekiq_workers config: <<-YAML
+      :verbose: false
+      :concurrency: 4
+      :queues:
+        - a
+        - b
+        - c
+      :blocking:
+        - [a, b]
+    YAML
+  end
+  def with_sidekiq_config(config)
+    whitespace_offset = config[/\A */].size
+    config.gsub!(/^ {#{whitespace_offset}}/, '')
+
+    puts "=> Use sidekiq config:\n#{config}"
+    File.write 'config/sidekiq.yml', config
+    yield
+  ensure
+    FileUtils.rm 'config/sidekiq.yml'
+  end
+
+  def run_sidekiq_monitoring
+    require 'sidekiq/web'
+    Thread.new do
+      Rack::Server.start app: Sidekiq::Web, Port: 3000
+    end
+    sleep 1
+    Launchy.open 'http://127.0.0.1:3000/busy?poll=true'
+  end
+
+  def run_sidekiq_workers(options)
+    require 'sidekiq/cli'
+    cli = Sidekiq::CLI.instance
+
+    %w[validate! boot_system].each do |stub|
+      cli.define_singleton_method(stub) {} # rubocop:disable Lint/EmptyBlock
+    end
+
+    with_sidekiq_config options[:config] do
+      cli.send :setup_options, []
+    end
+
+    cli.run
+  end
+end

data/demo/app/workers/a_worker.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class AWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :a
+
+  def perform
+    sleep 10
+  end
+end

data/demo/app/workers/b_worker.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class BWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :b
+
+  def perform
+    sleep 10
+  end
+end

data/demo/app/workers/c_worker.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class CWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :c
+
+  def perform
+    sleep 10
+  end
+end

data/demo/app/workers/fast_worker.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class FastWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :fast
+
+  def perform
+    sleep 0.2
+  end
+end

data/demo/app/workers/slow_worker.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class SlowWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :slow
+
+  def perform
+    sleep 1
+  end
+end

data/demo/config/application.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require File.expand_path('boot', __dir__)
+
+require 'action_controller/railtie'
+require 'action_mailer/railtie'
+require 'sprockets/railtie'
+
+Bundler.require(:default, Rails.env)
+
+module Demo
+  Application = Class.new Rails::Application
+end

data/demo/config/boot.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])

data/demo/config/environment.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require File.expand_path('application', __dir__)
+Demo::Application.initialize!

data/demo/config/environments/development.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+Demo::Application.configure do
+  config.cache_classes = false
+  config.eager_load = false
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+  config.action_mailer.raise_delivery_errors = false
+  config.active_support.deprecation = :log
+  config.assets.debug = true
+end

data/lib/sidekiq/extensions/manager.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class Manager
+    module InitLimitFetch
+      def initialize(capsule)
+        capsule.config[:fetch_class] = Sidekiq::LimitFetch
+        super
+      end
+
+      def start
+        # In sidekiq 6.5.0 the variable @options has been renamed to @config
+        Sidekiq::LimitFetch::Queues.start @config
+        Sidekiq::LimitFetch::Global::Monitor.start!
+        super
+      end
+    end
+
+    prepend InitLimitFetch
+  end
+end

data/lib/sidekiq/extensions/queue.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class Queue
+    extend Forwardable
+    extend LimitFetch::Instances
+    attr_reader :rname
+
+    def_delegators :lock,
+                   :limit,         :limit=, :limit_changed?,
+                   :process_limit, :process_limit=,
+                   :acquire,       :release,
+                   :pause,         :pause_for_ms, :unpause,
+                   :block,         :unblock,
+                   :paused?,       :blocking?,
+                   :unblocked,     :block_except,
+                   :probed,        :busy,
+                   :increase_busy, :decrease_busy,
+                   :local_busy?,   :explain,
+                   :remove_locks_except!,
+                   :clear_limits
+
+    def lock
+      @lock ||= LimitFetch::Global::Semaphore.new name
+    end
+  end
+end