sidekiq 0.11.0 → 0.11.1

data/Changes.md

@@ -1,6 +1,14 @@
 HEAD
 -----------
 
+- Fix fetch breaking retry when used with Redis namespaces. [#109]
+- Redis connection now just a plain ConnectionPool, not CP::Wrapper.
+- Capistrano initial deploy fix [#106]
+- Re-implemented weighted queues support (ryanlecompte)
+
+0.11.0
+-----------
+
 - Client-side API changes, added sidekiq\_options for Sidekiq::Worker.
   As a side effect of this change, the client API works on Ruby 1.8.
   It's not officially supported but should work [#103]

data/examples/config.yml

@@ -5,5 +5,5 @@
 :verbose:      false
 :concurrency:  25
 :queues:
-  - often
-  - seldom
+  - [often, 7]
+  - [seldom, 3]

data/lib/sidekiq/capistrano.rb

@@ -8,12 +8,12 @@ Capistrano::Configuration.instance.load do
 
     desc "Quiet sidekiq (stop accepting new work)"
     task :quiet do
-      run "cd #{current_path} && bundle exec sidekiqctl quiet #{current_path}/tmp/pids/sidekiq.pid"
+      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then bundle exec sidekiqctl quiet #{current_path}/tmp/pids/sidekiq.pid ; fi"
     end
 
     desc "Stop sidekiq"
     task :stop do
-      run "cd #{current_path} && bundle exec sidekiqctl stop #{current_path}/tmp/pids/sidekiq.pid #{fetch :sidekiq_timeout}"
+      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then bundle exec sidekiqctl stop #{current_path}/tmp/pids/sidekiq.pid #{fetch :sidekiq_timeout} ; fi"
     end
 
     desc "Start sidekiq"

data/lib/sidekiq/cli.rb

@@ -99,6 +99,7 @@ module Sidekiq
 
     def validate!
       options[:queues] << 'default' if options[:queues].empty?
+      options[:queues].shuffle!
 
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
@@ -115,8 +116,9 @@ module Sidekiq
       opts = {}
 
       @parser = OptionParser.new do |o|
-        o.on "-q", "--queue QUEUE", "Queue to process" do |arg|
-          parse_queues(opts, arg)
+         o.on "-q", "--queue QUEUE,WEIGHT", "Queue to process, with optional weight" do |arg|
+          q, weight = arg.split(",")
+          parse_queues(opts, q, weight)
         end
 
         o.on "-v", "--verbose", "Print more verbose output" do
@@ -175,14 +177,15 @@ module Sidekiq
       if cli[:config_file] && File.exist?(cli[:config_file])
         opts = YAML.load_file cli[:config_file]
         queues = opts.delete(:queues) || []
-        queues.each { |name, _| parse_queues(opts, name) }
+        queues.each { |name, weight| parse_queues(opts, name, weight) }
       end
       opts
     end
 
-    def parse_queues(opts, q)
-      (opts[:queues] ||= []) << q
+    def parse_queues(opts, q, weight)
+      (weight || 1).to_i.times do
+       (opts[:queues] ||= []) << q
+      end
     end
-
   end
 end

data/lib/sidekiq/fetch.rb

@@ -10,14 +10,13 @@ module Sidekiq
     include Celluloid
     include Sidekiq::Util
 
+    # Timeout for Redis#blpop.
     TIMEOUT = 1
 
     def initialize(mgr, queues)
-      @cmd = queues.map { |q| "queue:#{q}" }
       @mgr = mgr
-
-      # One second timeout for blpop.
-      @cmd << TIMEOUT
+      @queues = queues.map { |q| "queue:#{q}" }
+      @num_queues = queues.uniq.size
     end
 
     # Fetching is straightforward: the Manager makes a fetch
@@ -32,15 +31,27 @@ module Sidekiq
       watchdog('Fetcher#fetch died') do
         queue = nil
         msg = nil
-        Sidekiq.redis { |conn| (queue, msg) = conn.blpop(*@cmd) }
+        Sidekiq.redis { |conn| queue, msg = conn.blpop(*queues_cmd) }
 
         if msg
-          @mgr.assign!(msg, queue.gsub(/\Aqueue:/, ''))
+          @mgr.assign!(msg, queue.gsub(/.*queue:/, ''))
         else
           after(0) { fetch }
         end
       end
     end
 
+    private
+
+    # Creating the Redis#blpop command takes into account any
+    # configured queue weights. By default Redis#blpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#blpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      cmd = @queues.sample(@num_queues)
+      cmd << TIMEOUT
+      cmd
+    end
   end
 end

data/lib/sidekiq/redis_connection.rb

@@ -6,7 +6,8 @@ module Sidekiq
   class RedisConnection
     def self.create(options={})
       url = options[:url] || ENV['REDISTOGO_URL'] || 'redis://localhost:6379/0'
-      ConnectionPool::Wrapper.new(:timeout => 1, :size => (options[:size] || Sidekiq.options[:concurrency] || 25)) do
+      size = (options[:size] || Sidekiq.options[:concurrency] || 25)
+      ConnectionPool.new(:timeout => 1, :size => size) do
         build_client(url, options[:namespace])
       end
     end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "0.11.0"
+  VERSION = "0.11.1"
 end

data/test/config.yml

@@ -5,5 +5,5 @@
 :pidfile:      /tmp/sidekiq-config-test.pid
 :concurrency:  50
 :queues:
-  - often
-  - seldom
+  - [often, 2]
+  - [seldom, 1]

data/test/helper.rb

@@ -12,4 +12,4 @@ require 'sidekiq/util'
 Sidekiq::Util.logger.level = Logger::ERROR
 
 require 'sidekiq/redis_connection'
-REDIS = Sidekiq::RedisConnection.create(:url => "redis://localhost/15")
+REDIS = Sidekiq::RedisConnection.create(:url => "redis://localhost/15", :namespace => 'testy')

data/test/test_cli.rb

@@ -45,9 +45,9 @@ class TestCli < MiniTest::Unit::TestCase
       assert_equal 30, Sidekiq.options[:timeout]
     end
 
-    it 'handles multiple queues' do
-      @cli.parse(['sidekiq', '-q', 'foo', '-q', 'bar', '-r', './test/fake_env.rb'])
-      assert_equal %w(foo bar), Sidekiq.options[:queues]
+    it 'handles multiple queues with weights' do
+      @cli.parse(['sidekiq', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
+      assert_equal %w(bar foo foo foo), Sidekiq.options[:queues].sort
     end
 
     it 'sets verbose' do
@@ -110,7 +110,8 @@ class TestCli < MiniTest::Unit::TestCase
       end
 
       it 'sets queues' do
-        assert_equal %w(often seldom), Sidekiq.options[:queues]
+        assert_equal 2, Sidekiq.options[:queues].count { |q| q == 'often' }
+        assert_equal 1, Sidekiq.options[:queues].count { |q| q == 'seldom' }
       end
     end
 
@@ -132,8 +133,8 @@ class TestCli < MiniTest::Unit::TestCase
                     '-c', '100',
                     '-r', @tmp_lib_path,
                     '-P', @tmp_path,
-                    '-q', 'often',
-                    '-q', 'seldom'])
+                    '-q', 'often,7',
+                    '-q', 'seldom,3'])
       end
 
       after do
@@ -158,7 +159,8 @@ class TestCli < MiniTest::Unit::TestCase
       end
 
       it 'sets queues' do
-        assert_equal %w(often seldom), Sidekiq.options[:queues]
+        assert_equal 7, Sidekiq.options[:queues].count { |q| q == 'often' }
+        assert_equal 3, Sidekiq.options[:queues].count { |q| q == 'seldom' }
       end
     end
   end

data/test/test_stats.rb

@@ -55,23 +55,25 @@ class TestStats < MiniTest::Unit::TestCase
       msg = { 'class' => DumbWorker.to_s, 'args' => [nil] }
       boss = MiniTest::Mock.new
 
-      assert_equal [], @redis.smembers('workers')
-      assert_equal 0, @redis.get('stat:failed').to_i
-      assert_equal 0, @redis.get('stat:processed').to_i
+      @redis.with do |conn|
+        assert_equal [], conn.smembers('workers')
+        assert_equal 0, conn.get('stat:failed').to_i
+        assert_equal 0, conn.get('stat:processed').to_i
 
-      processor = Sidekiq::Processor.new(boss)
-      assert_equal 1, @redis.smembers('workers').size
+        processor = Sidekiq::Processor.new(boss)
+        assert_equal 1, conn.smembers('workers').size
 
-      pstr = processor.to_s
-      assert_raises RuntimeError do
-        processor.process(msg, 'xyzzy')
-      end
+        pstr = processor.to_s
+        assert_raises RuntimeError do
+          processor.process(msg, 'xyzzy')
+        end
 
-      set = @redis.smembers('workers')
-      assert_equal 0, set.size
-      assert_equal 1, @redis.get('stat:failed').to_i
-      assert_equal 1, @redis.get('stat:processed').to_i
-      assert_equal nil, @redis.get("stat:processed:#{pstr}")
+        set = conn.smembers('workers')
+        assert_equal 0, set.size
+        assert_equal 1, conn.get('stat:failed').to_i
+        assert_equal 1, conn.get('stat:processed').to_i
+        assert_equal nil, conn.get("stat:processed:#{pstr}")
+      end
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.11.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-03 00:00:00.000000000 Z
+date: 2012-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70099857099140 !ruby/object:Gem::Requirement
+  requirement: &70115690910620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099857099140
+  version_requirements: *70115690910620
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70099857098700 !ruby/object:Gem::Requirement
+  requirement: &70115690910180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099857098700
+  version_requirements: *70115690910180
 - !ruby/object:Gem::Dependency
   name: connection_pool
-  requirement: &70099857098200 !ruby/object:Gem::Requirement
+  requirement: &70115690926040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.9.0
   type: :runtime
   prerelease: false
-  version_requirements: *70099857098200
+  version_requirements: *70115690926040
 - !ruby/object:Gem::Dependency
   name: celluloid
-  requirement: &70099857097700 !ruby/object:Gem::Requirement
+  requirement: &70115690925540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *70099857097700
+  version_requirements: *70115690925540
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70099857097320 !ruby/object:Gem::Requirement
+  requirement: &70115690925160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099857097320
+  version_requirements: *70115690925160
 - !ruby/object:Gem::Dependency
   name: minitest
-  requirement: &70099857113240 !ruby/object:Gem::Requirement
+  requirement: &70115690924700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70099857113240
+  version_requirements: *70115690924700
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70099857112820 !ruby/object:Gem::Requirement
+  requirement: &70115690924280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70099857112820
+  version_requirements: *70115690924280
 - !ruby/object:Gem::Dependency
   name: slim
-  requirement: &70099857112400 !ruby/object:Gem::Requirement
+  requirement: &70115690923860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70099857112400
+  version_requirements: *70115690923860
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70099857111980 !ruby/object:Gem::Requirement
+  requirement: &70115690923440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70099857111980
+  version_requirements: *70115690923440
 - !ruby/object:Gem::Dependency
   name: actionmailer
-  requirement: &70099857111480 !ruby/object:Gem::Requirement
+  requirement: &70115690922940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,10 +120,10 @@ dependencies:
         version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70099857111480
+  version_requirements: *70115690922940
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70099857110980 !ruby/object:Gem::Requirement
+  requirement: &70115690922440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,7 +131,7 @@ dependencies:
         version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70099857110980
+  version_requirements: *70115690922440
 description: Simple, efficient message processing for Ruby
 email:
 - mperham@gmail.com