sidekiq-limit_fetch 1.7 → 2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c5121de39e03363b90b64633028f65ed5cf6983
-  data.tar.gz: 935a28cd47181070776e71fe60db08165cab2ffe
+  metadata.gz: 674ddce501d3c50f771a21a1e7546513af0a2669
+  data.tar.gz: f6bd2050d3e7eea9b599fc1989680b77c3e7de85
 SHA512:
-  metadata.gz: 09f48f0e5dce160c809b4fdad660f35c0fe2ac27ee3f6d850d094fc2e995adc70c3915cefb2ff4bc3a3aea2785c4f2a5edf3ca65c0692aea923dfbcd8ccdb2f4
-  data.tar.gz: a4d226be6b1df6fad9ab438dfd60e8e29235c89540f209c3399c5dd3ef24781184e653af3ae7cdbc67d08104f3a8fa16d6a44084ca36ecf3780d14e024579d78
+  metadata.gz: e6e72377703afe1307173b6174c41bae6e4d819222431c061b2d16339a678e45968c3b51101f4a96e11253f8df17910b76a343870f1a306e322dafe7aab84e89
+  data.tar.gz: fc59c7b2b8c779556914a2d4aeea8282354df0a2e19154f99513a02ff8d013642b4e68fce46959d2b311a0a30d5c41bbc999d7a5a55cb7a949a8255cb9a1d716

data/README.md

@@ -14,6 +14,10 @@ Add this line to your application's Gemfile:
 
     gem 'sidekiq-limit_fetch'
 
+### Requirements
+
+redis 2.6 or newer
+
 ## Usage
 
 ### Limits

data/benchmarks/compare.rb

@@ -0,0 +1,49 @@
+require 'benchmark'
+require 'sidekiq/cli'
+
+total       = (ARGV.shift || 50).to_i
+concurrency = ARGV.shift || 1
+limit       = ARGV.shift
+
+if limit
+  limit = nil if limit == 'nil'
+
+  $:.unshift File.expand_path './lib'
+  require 'sidekiq-limit_fetch'
+  Sidekiq::Queue['inline'].limit = limit
+  Sidekiq.redis {|it| it.del 'limit_fetch:probed:inline' }
+end
+
+Sidekiq::Queue.new('inline').clear
+
+class FastJob
+  include Sidekiq::Worker
+  sidekiq_options queue: :inline
+
+  def perform(i)
+    puts "job N#{i} is finished"
+  end
+end
+
+class FinishJob
+  include Sidekiq::Worker
+  sidekiq_options queue: :inline
+
+  def perform
+    Process.kill 'INT', 0
+  end
+end
+
+total.times {|i| FastJob.perform_async i+1 }
+FinishJob.perform_async
+
+Sidekiq::CLI.instance.tap do |cli|
+  %w(validate! boot_system).each {|stub| cli.define_singleton_method(stub) {}}
+  cli.parse ['-q inline', '-q other', "-c #{concurrency}"]
+  puts Benchmark.measure {
+    begin
+      cli.run
+    rescue Exception
+    end
+  }
+end

data/lib/sidekiq-limit_fetch.rb

@@ -0,0 +1 @@
+require_relative 'sidekiq/limit_fetch'

data/lib/sidekiq/limit_fetch.rb

@@ -4,6 +4,7 @@ require 'forwardable'
 class Sidekiq::LimitFetch
   autoload :UnitOfWork, 'sidekiq/limit_fetch/unit_of_work'
 
+  require_relative 'limit_fetch/redis'
   require_relative 'limit_fetch/singleton'
   require_relative 'limit_fetch/queues'
   require_relative 'limit_fetch/global/semaphore'
@@ -11,6 +12,7 @@ class Sidekiq::LimitFetch
   require_relative 'limit_fetch/global/monitor'
   require_relative 'extensions/queue'
 
+  include Redis
   Sidekiq.options[:fetch] = self
 
   def self.bulk_requeue(jobs)
@@ -30,13 +32,13 @@ class Sidekiq::LimitFetch
   private
 
   def fetch_message
-    queue, _ = redis_blpop *@queues.acquire, Sidekiq::Fetcher::TIMEOUT
+    queue, _ = redis_brpop *@queues.acquire, Sidekiq::Fetcher::TIMEOUT
   ensure
     @queues.release_except queue
   end
 
-  def redis_blpop(*args)
+  def redis_brpop(*args)
     return if args.size < 2
-    Sidekiq.redis {|it| it.blpop *args }
+    redis {|it| it.brpop *args }
   end
 end

data/lib/sidekiq/limit_fetch/redis.rb

@@ -0,0 +1,13 @@
+module Sidekiq::LimitFetch::Redis
+  # prevent blocking of fetcher
+  # more bullet-proof and faster (O_O)
+  # than using Celluloid::IO
+  def redis
+    Sidekiq.redis do |redis|
+      begin
+        Celluloid::Future.new { yield redis }.value
+      end
+    end
+  rescue Celluloid::Task::TerminatedError
+  end
+end

data/sidekiq-limit_fetch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '1.7'
+  gem.version       = '2.0'
   gem.authors       = 'brainopia'
   gem.email         = 'brainopia@evilmartians.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'

data/spec/sidekiq/limit_fetch_spec.rb

@@ -4,8 +4,8 @@ describe Sidekiq::LimitFetch do
   before :each do
     Sidekiq.redis do |it|
       it.del 'queue:queue1'
-      it.rpush 'queue:queue1', 'task1'
-      it.rpush 'queue:queue1', 'task2'
+      it.lpush 'queue:queue1', 'task1'
+      it.lpush 'queue:queue1', 'task2'
       it.expire 'queue:queue1', 30
     end
   end
@@ -30,7 +30,7 @@ describe Sidekiq::LimitFetch do
     Sidekiq::Queue['queue2'].busy.should == 0
 
     work = subject.retrieve_work
-    work.message.should == 'task2'
+    work.message.should == 'task1'
 
     Sidekiq::Queue['queue1'].busy.should == 1
     Sidekiq::Queue['queue2'].busy.should == 0
@@ -42,6 +42,6 @@ describe Sidekiq::LimitFetch do
     Sidekiq::Queue['queue2'].busy.should == 0
 
     work = subject.retrieve_work
-    work.message.should == 'task1'
+    work.message.should == 'task2'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: '1.7'
+  version: '2.0'
 platform: ruby
 authors:
 - brainopia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-22 00:00:00.000000000 Z
+date: 2013-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -72,12 +72,15 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmarks/compare.rb
+- lib/sidekiq-limit_fetch.rb
 - lib/sidekiq/extensions/queue.rb
 - lib/sidekiq/limit_fetch.rb
 - lib/sidekiq/limit_fetch/global/monitor.rb
 - lib/sidekiq/limit_fetch/global/selector.rb
 - lib/sidekiq/limit_fetch/global/semaphore.rb
 - lib/sidekiq/limit_fetch/queues.rb
+- lib/sidekiq/limit_fetch/redis.rb
 - lib/sidekiq/limit_fetch/singleton.rb
 - lib/sidekiq/limit_fetch/unit_of_work.rb
 - sidekiq-limit_fetch.gemspec
@@ -106,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.2
 signing_key: 
 specification_version: 4
 summary: Sidekiq strategy to support queue limits
@@ -117,4 +120,3 @@ test_files:
 - spec/sidekiq/limit_fetch/semaphore_spec.rb
 - spec/sidekiq/limit_fetch_spec.rb
 - spec/spec_helper.rb
-has_rdoc: