sidekiq-limit_fetch 0.2 → 0.3

data/README.md

@@ -19,9 +19,20 @@ Specify limits which you want to place on queues inside sidekiq.yml:
   queue_name2: 10
 ```
 
-In this example, tasks for the first restricted queue will be run by at most 5
-workers at the same time and the second queue will have no more than 10
+Or set it dynamically in your code:
+```ruby
+  Sidekiq::Queue.new('queue_name1').limit = 5
+  Sidekiq::Queue['queue_name2'].limit = 10
+```
+
+In these examples, tasks for the ```queue_name1``` will be run by at most 5
+workers at the same time and the ```queue_name2``` will have no more than 10
 workers simultaneously.
 
+Ability to set limits dynamically allows you to resize worker
+distribution among queues any time you want.
+
+Limits are applied strictly for current process.
+
 Sponsored by [Evil Martians].
 [Evil Martians]: http://evilmartians.com/

data/lib/sidekiq/limit_fetch.rb

@@ -3,11 +3,16 @@ require 'sidekiq/fetch'
 
 class Sidekiq::LimitFetch
   require_relative 'limit_fetch/semaphore'
-  require_relative 'limit_fetch/queue'
   require_relative 'limit_fetch/unit_of_work'
+  require_relative 'limit_fetch/singleton'
+  require_relative 'limit_fetch/queue'
 
   Sidekiq.options[:fetch] = self
 
+  def self.bulk_requeue(jobs)
+    Sidekiq::BasicFetch.bulk_requeue jobs
+  end
+
   def initialize(options)
     prepare_queues options
     options[:strict] ? define_strict_queues : define_weighted_queues
@@ -19,14 +24,18 @@ class Sidekiq::LimitFetch
 
   def retrieve_work
     queues = available_queues
+
+    if queues.empty?
+      sleep Sidekiq::Fetcher::TIMEOUT
+      return
+    end
+
     queue_name, message = Sidekiq.redis do |it|
       it.brpop *queues.map(&:full_name), Sidekiq::Fetcher::TIMEOUT
     end
 
     if message
-      queue = queues.find {|it| it.full_name == queue_name }
-      queues.delete queue
-
+      queue = queues.delete queues.find {|it| it.full_name == queue_name }
       UnitOfWork.new queue, message
     end
   ensure
@@ -36,11 +45,11 @@ class Sidekiq::LimitFetch
   private
 
   def prepare_queues(options)
-    cache = {}
     limits = options[:limits] || {}
-
     @queues = options[:queues].map do |name|
-      cache[name] ||= Queue.new name, limits[name]
+      Sidekiq::Queue.new(name).tap do |it|
+        it.limit = limits[name] if limits[name]
+      end
     end
   end
 

data/lib/sidekiq/limit_fetch/queue.rb

@@ -1,14 +1,14 @@
-class Sidekiq::LimitFetch
+module Sidekiq
   class Queue
-    extend Forwardable
+    extend LimitFetch::Singleton, Forwardable
+    def_delegators :lock, :limit, :limit=, :acquire, :release, :busy
 
-    attr_reader :name, :full_name
-    def_delegators :@lock, :acquire, :release
+    def full_name
+      @rname
+    end
 
-    def initialize(name, limit)
-      @name = name
-      @full_name = "queue:#{name}"
-      @lock = Semaphore.for limit
+    def lock
+      @lock ||= LimitFetch::Semaphore.new
     end
   end
 end

data/lib/sidekiq/limit_fetch/semaphore.rb

@@ -1,28 +1,26 @@
 class Sidekiq::LimitFetch::Semaphore
-  Stub = Struct.new(:acquire, :release)
+  attr_reader :limit, :busy
 
-  def self.for(limit)
-    limit ? new(limit) : stub
+  def initialize
+    @lock = Mutex.new
+    @busy = 0
   end
 
-  def self.stub
-    @stub ||= Stub.new(true, true)
-  end
-
-  def initialize(limit)
-    @lock  = Mutex.new
-    @limit = limit
+  def limit=(value)
+    @lock.synchronize do
+      @limit = value
+    end
   end
 
   def acquire
     @lock.synchronize do
-      @limit -= 1 if @limit > 0
+      @busy += 1 if not @limit or @limit > @busy
     end
   end
 
   def release
     @lock.synchronize do
-      @limit += 1
+      @busy -= 1
     end
   end
 end

data/lib/sidekiq/limit_fetch/singleton.rb

@@ -0,0 +1,11 @@
+module Sidekiq::LimitFetch::Singleton
+  def self.extended(klass)
+    klass.instance_variable_set :@instances, {}
+  end
+
+  def new(*args)
+    @instances[args] ||= super
+  end
+
+  alias [] new
+end

data/sidekiq-limit_fetch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '0.2'
+  gem.version       = '0.3'
   gem.authors       = 'brainopia'
   gem.email         = 'brainopia@evilmartians.com'
   gem.summary       = 'Sidekig strategy to support queue limits'

data/spec/{integration_spec.rb → limit_fetch_spec.rb}

@@ -1,11 +1,11 @@
-require 'sidekiq/limit_fetch'
+require 'spec_helper'
 
 describe Sidekiq::LimitFetch do
   before :each do
     Sidekiq.redis do |it|
-      it.del 'queue:example'
-      it.rpush 'queue:example', 'task'
-      it.expire 'queue:example', 30
+      it.del 'queue:example1'
+      it.rpush 'queue:example1', 'task'
+      it.expire 'queue:example1', 30
     end
   end
 
@@ -14,21 +14,21 @@ describe Sidekiq::LimitFetch do
   end
 
   def new_fetcher(options={})
-    described_class.new options.merge queues: %w(example example example2 example2)
+    described_class.new options.merge queues: %w(example1 example1 example2 example2)
   end
 
   it 'should retrieve weighted queues' do
     fetcher = new_fetcher
-    queues(fetcher).should =~ %w(queue:example queue:example2)
+    queues(fetcher).should =~ %w(queue:example1 queue:example2)
   end
 
   it 'should retrieve strictly ordered queues' do
     fetcher = new_fetcher strict: true
-    queues(fetcher).should == %w(queue:example queue:example2)
+    queues(fetcher).should == %w(queue:example1 queue:example2)
   end
 
-  it 'should retrieve limited queues' do
-    fetcher = new_fetcher strict: true, limits: { 'example' => 2 }
+  it 'should retrieve only available queues' do
+    fetcher = new_fetcher strict: true, limits: { 'example1' => 2 }
     queues = -> { fetcher.available_queues }
 
     queues1 = queues.call
@@ -47,11 +47,11 @@ describe Sidekiq::LimitFetch do
   end
 
   it 'should acquire lock on queue for excecution' do
-    fetcher = new_fetcher limits: { 'example' => 1, 'example2' => 1 }
+    fetcher = new_fetcher limits: { 'example1' => 1, 'example2' => 1 }
     work = fetcher.retrieve_work
     work.message.should == 'task'
-    work.queue.should == 'queue:example'
-    work.queue_name.should == 'example'
+    work.queue.should == 'queue:example1'
+    work.queue_name.should == 'example1'
 
     queues = fetcher.available_queues
     queues.should have(1).item
@@ -64,4 +64,18 @@ describe Sidekiq::LimitFetch do
 
     fetcher.available_queues.should have(2).items
   end
+
+  it 'should set queue limits on the fly' do
+    Sidekiq::Queue['example1'].limit = 1
+    Sidekiq::Queue['example2'].limit = 2
+
+    fetcher = new_fetcher
+
+    fetcher.available_queues.should have(2).item
+    fetcher.available_queues.should have(1).item
+    fetcher.available_queues.should have(0).item
+
+    Sidekiq::Queue['example1'].limit = 2
+    fetcher.available_queues.should have(1).item
+  end
 end

data/spec/sidekiq/limit_fetch/queue_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Sidekiq::Queue do
+  context 'singleton' do
+    shared_examples :constructor do
+      it 'with default name' do
+        new_object = -> { described_class.send constructor }
+        new_object.call.should == new_object.call
+      end
+
+      it 'with given name' do
+        new_object = ->(name) { described_class.send constructor, name }
+        new_object.call('name').should == new_object.call('name')
+      end
+    end
+
+    context '.new' do
+      let(:constructor) { :new }
+      it_behaves_like :constructor
+    end
+
+    context '.[]' do
+      let(:constructor) { :[] }
+      it_behaves_like :constructor
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'sidekiq/limit_fetch'
+
+RSpec.configure do |config|
+  config.before :each do
+    Sidekiq::Queue.instance_variable_set :@instances, {}
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.3'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-14 00:00:00.000000000 Z
+date: 2013-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -58,9 +58,12 @@ files:
 - lib/sidekiq/limit_fetch.rb
 - lib/sidekiq/limit_fetch/queue.rb
 - lib/sidekiq/limit_fetch/semaphore.rb
+- lib/sidekiq/limit_fetch/singleton.rb
 - lib/sidekiq/limit_fetch/unit_of_work.rb
 - sidekiq-limit_fetch.gemspec
-- spec/integration_spec.rb
+- spec/limit_fetch_spec.rb
+- spec/sidekiq/limit_fetch/queue_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/brainopia/sidekiq-limit_fetch
 licenses: []
 post_install_message: 
@@ -86,4 +89,6 @@ signing_key:
 specification_version: 3
 summary: Sidekig strategy to support queue limits
 test_files:
-- spec/integration_spec.rb
+- spec/limit_fetch_spec.rb
+- spec/sidekiq/limit_fetch/queue_spec.rb
+- spec/spec_helper.rb