sidekiq-limit_fetch 1.1 → 1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 49c36141cff8fd8e6ad664f6a2fbe07af64f21ce
+  data.tar.gz: bcd3045d1127d419892b6d917363cfaf6462c9f3
+SHA512:
+  metadata.gz: 014d8dd3ff1dfedbf226c0e5fc58fdb57a34e0329b0ad56f3d607276d39c45317697fde7c13cfeb33efa86c16a14db8df7a30eb3103f70691dd93c183ba15201
+  data.tar.gz: a6c5d5c6c19a03fb727384dee9ac0119ae018ae0ce517922442b4f45dcc576cb7f9566385af3b1d6a5696f57837b0b3475796c0338f1cd38883bbf85bd5ed18b

data/.travis.yml

@@ -8,5 +8,5 @@ services:
   - redis-server
 matrix:
   allow_failures:
-    - rvm: 2.0.0
+    - rvm: jruby-19mode
     - rvm: rbx-19mode

data/Gemfile

@@ -1,2 +1,2 @@
-source :rubygems
+source 'https://rubygems.org'
 gemspec

data/README.md

@@ -3,14 +3,10 @@
 Sidekiq strategy to restrict number of workers
 which are able to run specified queues simultaneously.
 
-[![Build
-Status](https://secure.travis-ci.org/brainopia/sidekiq-limit_fetch.png)](http://travis-ci.org/brainopia/sidekiq-limit_fetch)
-[![Gem
-Version](https://badge.fury.io/rb/sidekiq-limit_fetch.png)](http://badge.fury.io/rb/sidekiq-limit_fetch)
-[![Dependency
-Status](https://gemnasium.com/brainopia/sidekiq-limit_fetch.png)](https://gemnasium.com/brainopia/sidekiq-limit_fetch)
-[![Code
-Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch)
+[![Build Status](https://secure.travis-ci.org/brainopia/sidekiq-limit_fetch.png)](http://travis-ci.org/brainopia/sidekiq-limit_fetch)
+[![Gem Version](https://badge.fury.io/rb/sidekiq-limit_fetch.png)](http://badge.fury.io/rb/sidekiq-limit_fetch)
+[![Dependency Status](https://gemnasium.com/brainopia/sidekiq-limit_fetch.png)](https://gemnasium.com/brainopia/sidekiq-limit_fetch)
+[![Code Climate](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch.png)](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch)
 
 ## Installation
 
@@ -43,6 +39,13 @@ workers simultaneously.
 Ability to set limits dynamically allows you to resize worker
 distribution among queues any time you want.
 
+### Busy workers by queue
+
+You can see how many workers currently handling a queue:
+
+```ruby
+  Sidekiq::Queue['name'].busy # number of busy workers
+```
 
 ### Pauses
 
@@ -55,13 +58,6 @@ will be preserved.
   Sidekiq::Queue['name'].unpause # allows workers to use the queue
 ```
 
-
-You can see how many workers currently handling a queue:
-
-```ruby
-  Sidekiq::Queue['name'].busy # number of busy workers
-```
-
 ### Multiple processes
 
 Limits are applied per process. In case you have several worker
@@ -106,6 +102,29 @@ You can also enable and disable blocking mode for queues on the fly:
   Sidekiq::Queue['name'].unblock
 ```
 
+### Advanced blocking queues
+
+You can also block on array of queues. It means when any of them is
+running only queues higher and queues from their blocking group can
+run. It will be easier to understand with an example:
+
+```yaml
+  :queues:
+    - a
+    - b
+    - c
+    - d
+  :blocking:
+    - [b, c]
+```
+
+In this case tasks from `d` will be blocked when a task from queue `b` or `c` is executed.
+
+You can dynamically set exceptions for queue blocking:
+
+```ruby
+  Sidekiq::Queue['queue1'].block_except 'queue2'
+```
 
 ### Thanks
 

data/lib/sidekiq/extensions/queue.rb

@@ -3,11 +3,12 @@ module Sidekiq
     extend LimitFetch::Singleton, Forwardable
 
     def_delegators :lock,
-      :limit,   :limit=,
-      :acquire, :release,
-      :pause,   :unpause,
-      :block,   :unblock,
-      :paused?, :blocking?,
+      :limit,     :limit=,
+      :acquire,   :release,
+      :pause,     :unpause,
+      :block,     :unblock,
+      :paused?,   :blocking?,
+      :unblocked, :block_except,
       :busy
 
     def lock

data/lib/sidekiq/limit_fetch/global/selector.rb

@@ -11,7 +11,13 @@ module Sidekiq::LimitFetch::Global
     end
 
     def uuid
-      @uuid ||= SecureRandom.uuid
+      # - if we'll remove "@uuid ||=" from inside of mutex
+      # then @uuid can be overwritten
+      # - if we'll remove "@uuid ||=" from outside of mutex
+      # then each read will lead to mutex
+      @uuid ||= Thread.exclusive do
+        @uuid ||= SecureRandom.uuid
+      end
     end
 
     private
@@ -48,11 +54,12 @@ module Sidekiq::LimitFetch::Global
         local worker_name = table.remove(ARGV, 1)
         local queues      = ARGV
         local available   = {}
+        local unblocked   = {}
         local queue_locks
-        local blocked
+        local blocking_mode
 
         for _, queue in ipairs(queues) do
-          if not blocked then
+          if not blocking_mode or unblocked[queue] then
             local busy_key    = namespace..'busy:'..queue
             local pause_key   = namespace..'pause:'..queue
             local paused      = redis.call('get', pause_key)
@@ -68,7 +75,13 @@ module Sidekiq::LimitFetch::Global
                 queue_locks = redis.call('llen', busy_key)
               end
 
-              blocked = can_block and queue_locks > 0
+              blocking_mode = can_block and queue_locks > 0
+
+              if blocking_mode and can_block ~= 'true' then
+                for unblocked_queue in string.gmatch(can_block, "[^,]+") do
+                  unblocked[unblocked_queue] = true
+                end
+              end
 
               if not queue_limit or queue_limit > queue_locks then
                 redis.call('rpush', busy_key, worker_name)

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -46,6 +46,11 @@ module Sidekiq::LimitFetch::Global
       redis {|it| it.set "#{PREFIX}:block:#@name", true }
     end
 
+    def block_except(*queues)
+      raise ArgumentError if queues.empty?
+      redis {|it| it.set "#{PREFIX}:block:#@name", queues.join(',') }
+    end
+
     def unblock
       redis {|it| it.del "#{PREFIX}:block:#@name" }
     end

data/lib/sidekiq/limit_fetch/local/selector.rb

@@ -3,10 +3,17 @@ module Sidekiq::LimitFetch::Local
     extend self
 
     def acquire(names)
-      blocked = false
+      blocked   = false
+      unblocked = []
+
       queues(names).select {|queue|
-        next false      if blocked
-        blocked = true  if not queue.paused? and queue.blocking? and queue.busy > 0
+        next false if blocked and not unblocked.include?(queue.name)
+
+        if not queue.paused? and queue.blocking? and queue.busy > 0
+          blocked   = true
+          unblocked = queue.unblocked || []
+        end
+
         queue.acquire
       }.map(&:name)
     end

data/lib/sidekiq/limit_fetch/local/semaphore.rb

@@ -1,6 +1,6 @@
 module Sidekiq::LimitFetch::Local
   class Semaphore
-    attr_reader :limit, :busy
+    attr_reader :limit, :busy, :unblocked
 
     def initialize(name)
       @name = name
@@ -44,6 +44,12 @@ module Sidekiq::LimitFetch::Local
       @block = true
     end
 
+    def block_except(*queues)
+      raise ArgumentError if queues.empty?
+      @unblocked = queues
+      @block = true
+    end
+
     def unblock
       @block = false
     end

data/lib/sidekiq/limit_fetch/queues.rb

@@ -36,8 +36,12 @@ class Sidekiq::LimitFetch
     end
 
     def set_blocks(blocks)
-      blocks.to_a.each do |name|
-        Sidekiq::Queue[name].block
+      blocks.to_a.each do |it|
+        if it.is_a? Array
+          it.each {|name| Sidekiq::Queue[name].block_except it }
+        else
+          Sidekiq::Queue[it].block
+        end
       end
     end
 

data/sidekiq-limit_fetch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '1.1'
+  gem.version       = '1.2'
   gem.authors       = 'brainopia'
   gem.email         = 'brainopia@evilmartians.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'

data/spec/sidekiq/limit_fetch/queues_spec.rb

@@ -40,6 +40,18 @@ describe Sidekiq::LimitFetch::Queues do
       Sidekiq::Queue['queue2'].busy.should == 1
     end
 
+    it 'should block except given queues' do
+      Sidekiq::Queue['queue1'].block_except 'queue2'
+      subject.acquire
+      Sidekiq::Queue['queue1'].busy.should == 1
+      Sidekiq::Queue['queue2'].busy.should == 1
+
+      Sidekiq::Queue['queue1'].block_except 'queue404'
+      subject.acquire
+      Sidekiq::Queue['queue1'].busy.should == 2
+      Sidekiq::Queue['queue2'].busy.should == 1
+    end
+
     it 'should release queues' do
       subject.acquire
       subject.release_except nil

data/spec/sidekiq/limit_fetch_spec.rb

@@ -14,7 +14,7 @@ describe Sidekiq::LimitFetch do
   let(:options) {{ queues: queues, limits: limits, global: global }}
   let(:queues) { %w(queue1 queue1 queue2 queue2) }
   let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
-  
+
   shared_examples_for :strategy do
     it 'should acquire lock on queue for execution' do
       work = subject.retrieve_work

metadata

@@ -1,22 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: '1.1'
-  prerelease: 
+  version: '1.2'
 platform: ruby
 authors:
 - brainopia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-28 00:00:00.000000000 Z
+date: 2013-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.6.5
     - - <
@@ -25,9 +23,8 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.6.5
     - - <
@@ -36,37 +33,34 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: ! "    Sidekiq strategy to restrict number of workers\n    which are
-  able to run specified queues simultaneously.\n"
+description: |2
+      Sidekiq strategy to restrict number of workers
+      which are able to run specified queues simultaneously.
 email: brainopia@evilmartians.com
 executables: []
 extensions: []
@@ -97,33 +91,26 @@ files:
 - spec/spec_helper.rb
 homepage: https://github.com/brainopia/sidekiq-limit_fetch
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3014276428821981336
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3014276428821981336
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Sidekiq strategy to support queue limits
 test_files:
 - spec/sidekiq/extensions/queue_spec.rb