rcgt-sidekiq-limit_fetch 3.4.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e9ffb7cc431bd4e8626c7691d2203893ded60945489fab10c6ac4d18faf35a30
+  data.tar.gz: eec03e77491392d8e749f71202b2cada20f93accf29e98486cf9f3e31eeab3db
+SHA512:
+  metadata.gz: d7c752b55bb5aaaf9e73b1e6cc02c738ffddd0f61a10303f040756d914b109b5ac4f382d0b2d7098ab0121fb0947a341eaf6271026cd417d9d0178c5e04d950a
+  data.tar.gz: 7e8d5cb9932ba6539b6fabd66c9f6e1842818aa99e3af395fef991eec5cb704f10efd3490fb06ae310cb50d10acb8ac9e5baefe23d855b7345af944d544931e7

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+pkg/
+.bundle/

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper --color

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.3.1
+  - 2.2.5
+  - jruby-9.1.5.0
+services:
+  - redis-server

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'sidekiq'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 brainopia
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,161 @@
+## Description
+
+A fork of https://github.com/brainopia/sidekiq-limit_fetch
+
+Sidekiq strategy to support a granular queue control –
+limiting, pausing, blocking, querying.
+
+[![Build Status](https://secure.travis-ci.org/brainopia/sidekiq-limit_fetch.svg)](http://travis-ci.org/brainopia/sidekiq-limit_fetch)
+[![Gem Version](https://badge.fury.io/rb/sidekiq-limit_fetch.svg)](http://badge.fury.io/rb/sidekiq-limit_fetch)
+[![Dependency Status](https://gemnasium.com/brainopia/sidekiq-limit_fetch.svg)](https://gemnasium.com/brainopia/sidekiq-limit_fetch)
+[![Code Climate](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch.svg)](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rcgt-sidekiq-limit_fetch'
+
+### Requirements
+
+**Important note:** At this moment, `sidekiq-limit_fetch` is incompatible with
+- sidekiq pro's `reliable_fetch`
+- `sidekiq-rate-limiter`
+- any other plugin that rewrites fetch strategy of sidekiq.
+
+## Usage
+
+### Require
+You must `require 'sidekiq-limit_fetch'` if it isn't already. It will not work until then.
+
+### Limits
+
+Specify limits which you want to place on queues inside sidekiq.yml:
+
+```yaml
+  :limits:
+    queue_name1: 5
+    queue_name2: 10
+```
+
+Or set it dynamically in your code:
+```ruby
+  Sidekiq::Queue['queue_name1'].limit = 5
+  Sidekiq::Queue['queue_name2'].limit = 10
+```
+
+In these examples, tasks for the ```queue_name1``` will be run by at most 5
+workers at the same time and the ```queue_name2``` will have no more than 10
+workers simultaneously.
+
+Ability to set limits dynamically allows you to resize worker
+distribution among queues any time you want.
+
+### Limits per process
+
+If you use multiple sidekiq processes then you can specify limits per process:
+
+```yaml
+  :process_limits:
+    queue_name: 2
+```
+
+Or set it in your code:
+
+```ruby
+  Sidekiq::Queue['queue_name'].process_limit = 2
+```
+
+### Busy workers by queue
+
+You can see how many workers currently handling a queue:
+
+```ruby
+  Sidekiq::Queue['name'].busy # number of busy workers
+```
+
+### Pauses
+
+You can also pause your queues temporarily. Upon continuing their limits
+will be preserved.
+
+```ruby
+  Sidekiq::Queue['name'].pause # prevents workers from running tasks from this queue
+  Sidekiq::Queue['name'].paused? # => true
+  Sidekiq::Queue['name'].unpause # allows workers to use the queue
+  Sidekiq::Queue['name'].pause_for_ms(1000) # will pause for a second
+```
+
+### Blocking queue mode
+
+If you use strict queue ordering (it will be used if you don't specify queue weights)
+then you can set blocking status for queues. It means if a blocking
+queue task is executing then no new task from lesser priority queues will
+be ran. Eg,
+
+```yaml
+  :queues:
+    - a
+    - b
+    - c
+  :blocking:
+    - b
+```
+
+In this case when a task for `b` queue is ran no new task from `c` queue
+will be started.
+
+You can also enable and disable blocking mode for queues on the fly:
+
+```ruby
+  Sidekiq::Queue['name'].block
+  Sidekiq::Queue['name'].blocking? # => true
+  Sidekiq::Queue['name'].unblock
+```
+
+### Advanced blocking queues
+
+You can also block on array of queues. It means when any of them is
+running only queues higher and queues from their blocking group can
+run. It will be easier to understand with an example:
+
+```yaml
+  :queues:
+    - a
+    - b
+    - c
+    - d
+  :blocking:
+    - [b, c]
+```
+
+In this case tasks from `d` will be blocked when a task from queue `b` or `c` is executed.
+
+You can dynamically set exceptions for queue blocking:
+
+```ruby
+  Sidekiq::Queue['queue1'].block_except 'queue2'
+```
+
+### Dynamic queues
+
+You can support dynamic queues (that are not listed in sidekiq.yml but
+that have tasks pushed to them (usually with `Sidekiq::Client.push`)).
+
+To use this mode you need to specify a following line in sidekiq.yml:
+
+```yaml
+  :dynamic: true
+```
+
+Dynamic queues will be ran at the lowest priority.
+
+### Maintenance
+
+If you use ```flushdb```, restart the sidekiq process to re-populate the dynamic configuration.
+
+### Thanks
+
+<a href="https://evilmartians.com/?utm_source=sidekiq-limit_fetch">
+<img src="https://evilmartians.com/badges/sponsored-by-evil-martians.svg" alt="Sponsored by Evil Martians" width="236" height="54"></a>
+ 

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new
+
+task :default do
+  rspec = Rake::Task[:spec]
+  rspec.invoke
+  ENV['namespace'] = 'namespace'
+  rspec.reenable
+  rspec.invoke
+end

data/bench/compare.rb

@@ -0,0 +1,52 @@
+require 'benchmark'
+require 'sidekiq/cli'
+require 'sidekiq/api'
+
+total       = (ARGV.shift || 50).to_i
+concurrency = ARGV.shift || 1
+limit       = ARGV.shift
+
+if limit
+  limit = nil if limit == 'nil'
+
+  $:.unshift File.expand_path '../lib'
+  require 'sidekiq-limit_fetch'
+  Sidekiq::Queue['inline'].limit = limit
+  Sidekiq.redis {|it| it.del 'limit_fetch:probed:inline' }
+  Sidekiq::LimitFetch::Queues.send(:define_method, :set) {|*| }
+end
+
+Sidekiq::Queue.new('inline').clear
+
+class FastJob
+  include Sidekiq::Worker
+  sidekiq_options queue: :inline
+
+  def perform(i)
+    puts "job N#{i} is finished"
+  end
+end
+
+class FinishJob
+  include Sidekiq::Worker
+  sidekiq_options queue: :inline
+
+  def perform
+    Process.kill 'INT', 0
+  end
+end
+
+total.times {|i| FastJob.perform_async i+1 }
+FinishJob.perform_async
+
+Sidekiq::CLI.instance.tap do |cli|
+  %w(validate! boot_system).each {|stub| cli.define_singleton_method(stub) {}}
+  cli.parse ['-q inline', '-q other', "-c #{concurrency}"]
+
+  puts Benchmark.measure {
+    begin
+      cli.run
+    rescue Exception
+    end
+  }
+end

data/demo/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem 'rails'
+gem 'launchy'
+gem 'sidekiq', github: 'mperham/sidekiq'
+gem 'sidekiq-limit_fetch', path: '..'
+

data/demo/README.md

@@ -0,0 +1,37 @@
+This is a demo rails app with a configured sidekiq-limit_fetch.
+
+Its purpose is to check whether plugin works in certain situations.
+
+Application is preconfigured with two workers:
+- `app/workers/fast_worker.rb` which does `sleep 0.2`
+- `app/workers/slow_worker.rb` which does `sleep 1`
+
+There is also a rake task which can be invoked as `bundle exec rake demo:limit`:
+
+- it prefills sidekiq tasks
+
+```ruby
+  100.times do
+    SlowWorker.perform_async
+    FastWorker.perform_async
+  end
+```
+- sets sidekiq config
+
+```yaml
+  :verbose: false
+  :concurrency: 4
+  :queues:
+    - slow
+    - fast
+  :limits:
+    slow: 1
+```
+
+- and launches a sidekiq admin page with overview of queues in browser.
+The page is set to live-poll so effects of limits can be seen directly.
+
+
+To change simulation modify `Rakefile` or workers.
+
+Any bugs related to the plugin should be demonstrated with a reproduction from this base app.

data/demo/Rakefile

@@ -0,0 +1,99 @@
+require File.expand_path('../config/application', __FILE__)
+Demo::Application.load_tasks
+
+namespace :demo do
+  task limit: :environment do
+    puts '=> Creating sidekiq tasks'
+
+    100.times do
+      SlowWorker.perform_async
+      FastWorker.perform_async
+    end
+
+    run_sidekiq_monitoring
+    run_sidekiq_workers config: <<-YAML
+      :verbose: false
+      :concurrency: 4
+      :queues:
+        - slow
+        - fast
+      :limits:
+        slow: 1
+    YAML
+  end
+
+  task blocking: :environment do
+    puts '=> Creating sidekiq tasks'
+
+    AWorker.perform_async
+    BWorker.perform_async
+    CWorker.perform_async
+
+    run_sidekiq_monitoring
+    run_sidekiq_workers config: <<-YAML
+      :verbose: false
+      :concurrency: 4
+      :queues:
+        - a
+        - b
+        - c
+      :blocking:
+        - a
+    YAML
+  end
+
+  task advanced_blocking: :environment do
+    puts '=> Creating sidekiq tasks'
+
+    AWorker.perform_async
+    BWorker.perform_async
+    CWorker.perform_async
+
+    run_sidekiq_monitoring
+    run_sidekiq_workers config: <<-YAML
+      :verbose: false
+      :concurrency: 4
+      :queues:
+        - a
+        - b
+        - c
+      :blocking:
+        - [a, b]
+    YAML
+  end
+  def with_sidekiq_config(config)
+    whitespace_offset = config[/\A */].size
+    config.gsub! /^ {#{whitespace_offset}}/, ''
+
+    puts "=> Use sidekiq config:\n#{config}"
+    File.write 'config/sidekiq.yml', config
+    yield
+  ensure
+    FileUtils.rm 'config/sidekiq.yml'
+  end
+
+  def run_sidekiq_monitoring
+    require 'sidekiq/web'
+    Thread.new do
+      Rack::Server.start app: Sidekiq::Web, Port: 3000
+    end
+    sleep 1
+    Launchy.open 'http://127.0.0.1:3000/busy?poll=true'
+  end
+
+  def run_sidekiq_workers(options)
+    require 'sidekiq/cli'
+    cli = Sidekiq::CLI.instance
+
+    %w(validate! boot_system).each do |stub|
+      cli.define_singleton_method(stub) {}
+    end
+
+    with_sidekiq_config options[:config] do
+      cli.send :setup_options, []
+    end
+
+    cli.run
+  end
+end
+

data/demo/app/workers/a_worker.rb

@@ -0,0 +1,8 @@
+class AWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :a
+
+  def perform
+    sleep 10
+  end
+end

data/demo/app/workers/b_worker.rb

@@ -0,0 +1,8 @@
+class BWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :b
+
+  def perform
+    sleep 10
+  end
+end

data/demo/app/workers/c_worker.rb

@@ -0,0 +1,9 @@
+class CWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :c
+
+  def perform
+    sleep 10
+  end
+end
+

data/demo/app/workers/fast_worker.rb

@@ -0,0 +1,8 @@
+class FastWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :fast
+
+  def perform
+    sleep 0.2
+  end
+end

data/demo/app/workers/slow_worker.rb

@@ -0,0 +1,8 @@
+class SlowWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :slow
+
+  def perform
+    sleep 1
+  end
+end

data/demo/config/application.rb

@@ -0,0 +1,11 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'action_controller/railtie'
+require 'action_mailer/railtie'
+require 'sprockets/railtie'
+
+Bundler.require(:default, Rails.env)
+
+module Demo
+  Application = Class.new Rails::Application
+end

data/demo/config/boot.rb

@@ -0,0 +1,2 @@
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])

data/demo/config/environment.rb

@@ -0,0 +1,2 @@
+require File.expand_path('../application', __FILE__)
+Demo::Application.initialize!

data/demo/config/environments/development.rb

@@ -0,0 +1,9 @@
+Demo::Application.configure do
+  config.cache_classes = false
+  config.eager_load = false
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+  config.action_mailer.raise_delivery_errors = false
+  config.active_support.deprecation = :log
+  config.assets.debug = true
+end

data/lib/sidekiq/extensions/manager.rb

@@ -0,0 +1,16 @@
+class Sidekiq::Manager
+  module InitLimitFetch
+    def initialize(options={})
+      options[:fetch] = Sidekiq::LimitFetch
+      super
+    end
+
+    def start
+      Sidekiq::LimitFetch::Queues.start options
+      Sidekiq::LimitFetch::Global::Monitor.start!
+      super
+    end
+  end
+
+  prepend InitLimitFetch
+end

data/lib/sidekiq/extensions/queue.rb

@@ -0,0 +1,23 @@
+module Sidekiq
+  class Queue
+    extend LimitFetch::Instances, Forwardable
+    attr_reader :rname
+
+    def_delegators :lock,
+      :limit,         :limit=,         :limit_changed?,
+      :process_limit, :process_limit=,
+      :acquire,       :release,
+      :pause,         :pause_for_ms,   :unpause,
+      :block,         :unblock,
+      :paused?,       :blocking?,
+      :unblocked,     :block_except,
+      :probed,        :busy,
+      :increase_busy, :decrease_busy,
+      :local_busy?,   :explain,
+      :remove_locks_except!
+
+    def lock
+      @lock ||= LimitFetch::Global::Semaphore.new name
+    end
+  end
+end

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -0,0 +1,72 @@
+module Sidekiq::LimitFetch::Global
+  module Monitor
+    extend self
+
+    HEARTBEAT_PREFIX = 'limit:heartbeat:'
+    PROCESS_SET = 'limit:processes'
+    HEARTBEAT_TTL = 20
+    REFRESH_TIMEOUT = 5
+
+    def start!(ttl=HEARTBEAT_TTL, timeout=REFRESH_TIMEOUT)
+      Thread.new do
+        loop do
+          Sidekiq::LimitFetch.redis_retryable do
+            add_dynamic_queues
+            update_heartbeat ttl
+            invalidate_old_processes
+          end
+
+          sleep timeout
+        end
+      end
+    end
+
+    def all_processes
+      Sidekiq.redis {|it| it.smembers PROCESS_SET }
+    end
+
+    def old_processes
+      all_processes.reject do |process|
+        Sidekiq.redis {|it| it.get heartbeat_key process }
+      end
+    end
+
+    def remove_old_processes!
+      Sidekiq.redis do |it|
+        old_processes.each {|process| it.srem PROCESS_SET, process }
+      end
+    end
+
+    def add_dynamic_queues
+      queues = Sidekiq::LimitFetch::Queues
+      queues.add Sidekiq::Queue.all.map(&:name) if queues.dynamic?
+    end
+
+    private
+
+    def update_heartbeat(ttl)
+      Sidekiq.redis do |it|
+        it.multi do
+          it.set heartbeat_key, true
+          it.sadd PROCESS_SET, Selector.uuid
+          it.expire heartbeat_key, ttl
+        end
+      end
+    end
+
+    def invalidate_old_processes
+      Sidekiq.redis do |it|
+        remove_old_processes!
+        processes = all_processes
+
+        Sidekiq::Queue.instances.each do |queue|
+          queue.remove_locks_except! processes
+        end
+      end
+    end
+
+    def heartbeat_key(process=Selector.uuid)
+      HEARTBEAT_PREFIX + process
+    end
+  end
+end