sidekiq-limit_fetch 0.7 → 0.8

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+  - rbx-19mode
+services:
+  - redis-server
+matrix:
+  allow_failures:
+    - rvm: 2.0.0
+    - rvm: rbx-19mode

data/README.md

@@ -3,6 +3,15 @@
 Sidekiq strategy to restrict number of workers
 which are able to run specified queues simultaneously.
 
+[![Build
+Status](https://secure.travis-ci.org/brainopia/sidekiq-limit_fetch.png)](http://travis-ci.org/brainopia/sidekiq-limit_fetch)
+[![Gem
+Version](https://badge.fury.io/rb/sidekiq-limit_fetch.png)](http://badge.fury.io/rb/sidekiq-limit_fetch)
+[![Dependency
+Status](https://gemnasium.com/brainopia/sidekiq-limit_fetch.png)](https://gemnasium.com/brainopia/sidekiq-limit_fetch)
+[![Code
+Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -14,9 +23,9 @@ Add this line to your application's Gemfile:
 Specify limits which you want to place on queues inside sidekiq.yml:
 
 ```yaml
-:limits:
-  queue_name1: 5
-  queue_name2: 10
+  :limits:
+    queue_name1: 5
+    queue_name2: 10
 ```
 
 Or set it dynamically in your code:
@@ -32,22 +41,58 @@ workers simultaneously.
 Ability to set limits dynamically allows you to resize worker
 distribution among queues any time you want.
 
+
 You can also pause your queues temporarely. Upon continuing their limits
 will be preserved.
 
 ```ruby
   Sidekiq::Queue['name'].pause # prevents workers from running tasks from this queue
-  ...
-  Sidekiq::Queue['name'].continue # allows workers to use the queue with the same limit
+  Sidekiq::Queue['name'].paused? # => true
+  Sidekiq::Queue['name'].unpause # allows workers to use the queue
+```
+
+
+You can see how many workers currently handling a queue:
+
+```ruby
+  Sidekiq::Queue['name'].busy # number of busy workers
 ```
 
+
 Limits are applied per process. In case you have several worker
 processes and want to have global locks between them, you'll need to
 enable global mode by setting global option, eg:
 
 ```yaml
-:global: true
+  :global: true
 ```
 
+
+If you use strict queue ordering (it will be used if you don't specify queue weights)
+then you can set blocking status for queues. It means if a blocking
+queue task is executing then no new task from lesser priority queues will
+be ran. Eg,
+
+```yaml
+  :queues:
+    - a
+    - b
+    - c
+  :limits:
+    - b
+```
+
+In this case when a task for `b` queue is ran no new task from `c` queue
+will be started.
+
+You can also enable and disable blocking mode for queues on the fly:
+
+```ruby
+  Sidekiq::Queue['name'].block
+  Sidekiq::Queue['name'].blocking? # => true
+  Sidekiq::Queue['name'].unblock
+```
+
+
 Sponsored by [Evil Martians].
 [Evil Martians]: http://evilmartians.com/

data/Rakefile

@@ -1 +1,5 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new
+task default: :spec

data/lib/sidekiq/extensions/queue.rb

@@ -5,7 +5,9 @@ module Sidekiq
     def_delegators :lock,
       :limit,   :limit=,
       :acquire, :release,
-      :pause,   :continue,
+      :pause,   :unpause,
+      :block,   :unblock,
+      :paused?, :blocking?,
       :busy
 
     def lock

data/lib/sidekiq/limit_fetch/global/selector.rb

@@ -48,26 +48,32 @@ module Sidekiq::LimitFetch::Global
         local worker_name = table.remove(ARGV, 1)
         local queues      = ARGV
         local available   = {}
+        local queue_locks
+        local blocked
 
         for _, queue in ipairs(queues) do
-          local busy_key    = namespace..'busy:'..queue
-          local pause_key   = namespace..'pause:'..queue
-          local paused      = redis.call('get', pause_key)
+          if not blocked then
+            local busy_key    = namespace..'busy:'..queue
+            local pause_key   = namespace..'pause:'..queue
+            local paused      = redis.call('get', pause_key)
 
-          if not paused then
-            local limit_key   = namespace..'limit:'..queue
-            local queue_limit = tonumber(redis.call('get', limit_key))
+            if not paused then
+              local limit_key   = namespace..'limit:'..queue
+              local queue_limit = tonumber(redis.call('get', limit_key))
 
-            if queue_limit then
-              local queue_locks = redis.call('llen', busy_key)
+              local block_key = namespace..'block:'..queue
+              local can_block = redis.call('get', block_key)
 
-              if queue_limit > queue_locks then
+              if can_block or queue_limit then
+                queue_locks = redis.call('llen', busy_key)
+              end
+
+              blocked = can_block and queue_locks > 0
+
+              if not queue_limit or queue_limit > queue_locks then
                 redis.call('rpush', busy_key, worker_name)
                 table.insert(available, queue)
               end
-            else
-              redis.call('rpush', busy_key, worker_name)
-              table.insert(available, queue)
             end
           end
         end

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -1,5 +1,8 @@
 module Sidekiq::LimitFetch::Global
   class Semaphore
+    extend Forwardable
+    def_delegator Sidekiq, :redis
+
     PREFIX = 'limit_fetch'
 
     def initialize(name)
@@ -7,12 +10,12 @@ module Sidekiq::LimitFetch::Global
     end
 
     def limit
-      value = Sidekiq.redis {|it| it.get "#{PREFIX}:limit:#@name" }
+      value = redis {|it| it.get "#{PREFIX}:limit:#@name" }
       value.to_i if value
     end
 
     def limit=(value)
-      Sidekiq.redis {|it| it.set "#{PREFIX}:limit:#@name", value }
+      redis {|it| it.set "#{PREFIX}:limit:#@name", value }
     end
 
     def acquire
@@ -24,15 +27,31 @@ module Sidekiq::LimitFetch::Global
     end
 
     def busy
-      Sidekiq.redis {|it| it.llen "#{PREFIX}:busy:#@name" }
+      redis {|it| it.llen "#{PREFIX}:busy:#@name" }
     end
 
     def pause
-      Sidekiq.redis {|it| it.set "#{PREFIX}:pause:#@name", true }
+      redis {|it| it.set "#{PREFIX}:pause:#@name", true }
+    end
+
+    def unpause
+      redis {|it| it.del "#{PREFIX}:pause:#@name" }
+    end
+
+    def paused?
+      redis {|it| it.get "#{PREFIX}:pause:#@name" }
+    end
+
+    def block
+      redis {|it| it.set "#{PREFIX}:block:#@name", true }
+    end
+
+    def unblock
+      redis {|it| it.del "#{PREFIX}:block:#@name" }
     end
 
-    def continue
-      Sidekiq.redis {|it| it.del "#{PREFIX}:pause:#@name" }
+    def blocking?
+      redis {|it| it.get "#{PREFIX}:block:#@name" }
     end
   end
 end

data/lib/sidekiq/limit_fetch/local/selector.rb

@@ -3,7 +3,12 @@ module Sidekiq::LimitFetch::Local
     extend self
 
     def acquire(names)
-      queues(names).select(&:acquire).map(&:name)
+      blocked = false
+      queues(names).select {|queue|
+        next false      if blocked
+        blocked = true  if not queue.paused? and queue.blocking? and queue.busy > 0
+        queue.acquire
+      }.map(&:name)
     end
 
     def release(names)

data/lib/sidekiq/limit_fetch/local/semaphore.rb

@@ -32,8 +32,24 @@ module Sidekiq::LimitFetch::Local
       @paused = true
     end
 
-    def continue
+    def unpause
       @paused = false
     end
+
+    def paused?
+      @paused
+    end
+
+    def block
+      @block = true
+    end
+
+    def unblock
+      @block = false
+    end
+
+    def blocking?
+      @block
+    end
   end
 end

data/sidekiq-limit_fetch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '0.7'
+  gem.version       = '0.8'
   gem.authors       = 'brainopia'
   gem.email         = 'brainopia@evilmartians.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'
@@ -14,6 +14,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep %r{^spec/}
   gem.require_paths = %w(lib)
 
-  gem.add_dependency 'sidekiq', '>= 2.6.5'
+  gem.add_dependency 'sidekiq', '>= 2.6.5', '< 3'
   gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rake'
 end

data/spec/sidekiq/extensions/queue_spec.rb

@@ -40,7 +40,7 @@ describe Sidekiq::Queue do
 
         it 'should be continuable' do
           queue.pause
-          queue.continue
+          queue.unpause
           queue.acquire.should be
         end
 
@@ -69,6 +69,22 @@ describe Sidekiq::Queue do
           queue.release
           queue.busy.should == 0
         end
+
+        it 'should tell if paused' do
+          queue.should_not be_paused
+          queue.pause
+          queue.should be_paused
+          queue.unpause
+          queue.should_not be_paused
+        end
+
+        it 'should tell if blocking' do
+          queue.should_not be_blocking
+          queue.block
+          queue.should be_blocking
+          queue.unblock
+          queue.should_not be_blocking
+        end
       end
 
       context 'global' do

data/spec/sidekiq/limit_fetch/global/monitor_spec.rb

@@ -22,7 +22,7 @@ describe Sidekiq::LimitFetch::Global::Monitor do
       2.times { queue.acquire }
       described_class.send(:invalidate_old_processors)
       queue.busy.should == 2
-      sleep 2
+      sleep 4
       described_class.send(:invalidate_old_processors)
       queue.busy.should == 0
     end

data/spec/sidekiq/limit_fetch/queues_spec.rb

@@ -23,6 +23,18 @@ describe Sidekiq::LimitFetch::Queues do
       Sidekiq::Queue['queue2'].busy.should == 1
     end
 
+    it 'should acquire blocking queues' do
+      subject.acquire
+      Sidekiq::Queue['queue1'].busy.should == 1
+      Sidekiq::Queue['queue2'].busy.should == 1
+
+      Sidekiq::Queue['queue1'].block
+
+      subject.acquire
+      Sidekiq::Queue['queue1'].busy.should == 2
+      Sidekiq::Queue['queue2'].busy.should == 1
+    end
+
     it 'should release queues' do
       subject.acquire
       subject.release_except nil

data/spec/sidekiq/limit_fetch/semaphore_spec.rb

@@ -40,7 +40,7 @@ describe 'semaphore' do
     it 'should unpause tasks' do
       subject.pause
       3.times { subject.acquire }
-      subject.continue
+      subject.unpause
       2.times { subject.acquire }
       subject.busy.should == 2
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: '0.7'
+  version: '0.8'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-24 00:00:00.000000000 Z
+date: 2013-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -19,6 +19,9 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.6.5
+    - - <
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,6 +30,9 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.6.5
+    - - <
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -43,6 +49,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! "    Sidekiq strategy to restrict number of workers\n    which are
   able to run specified queues simultaneously.\n"
 email: brainopia@evilmartians.com
@@ -51,6 +73,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -84,12 +107,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 4297700822520772168
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 4297700822520772168
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24