sidekiq 1.1.2 → 1.1.3

data/Changes.md

@@ -1,3 +1,12 @@
+1.1.3
+-----------
+
+- Better network error handling when fetching jobs from Redis.
+  Sidekiq will retry once per second until it can re-establish
+  a connection. (ryanlecompte)
+- capistrano recipe now uses `bundle_cmd` if set [#147]
+- handle multi\_json API changes (sferik)
+
 1.1.2
 -----------
 

data/lib/sidekiq.rb

@@ -82,4 +82,22 @@ module Sidekiq
     @server_chain
   end
 
+  def self.load_json(string)
+    # Can't reliably detect whether MultiJson responds to load, since it's
+    # a reserved word. Use adapter as a proxy for new features.
+    if MultiJson.respond_to?(:adapter)
+      MultiJson.load(string)
+    else
+      MultiJson.decode(string)
+    end
+  end
+
+  def self.dump_json(object)
+    if MultiJson.respond_to?(:dump)
+      MultiJson.dump(object)
+    else
+      MultiJson.encode(object)
+    end
+  end
+
 end

data/lib/sidekiq/capistrano.rb

@@ -11,18 +11,18 @@ Capistrano::Configuration.instance.load do
 
     desc "Quiet sidekiq (stop accepting new work)"
     task :quiet, :roles => lambda { fetch(:sidekiq_role) } do
-      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then bundle exec sidekiqctl quiet #{current_path}/tmp/pids/sidekiq.pid ; fi"
+      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl quiet #{current_path}/tmp/pids/sidekiq.pid ; fi"
     end
 
     desc "Stop sidekiq"
     task :stop, :roles => lambda { fetch(:sidekiq_role) } do
-      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then bundle exec sidekiqctl stop #{current_path}/tmp/pids/sidekiq.pid #{fetch :sidekiq_timeout} ; fi"
+      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl stop #{current_path}/tmp/pids/sidekiq.pid #{fetch :sidekiq_timeout} ; fi"
     end
 
     desc "Start sidekiq"
     task :start, :roles => lambda { fetch(:sidekiq_role) } do
       rails_env = fetch(:rails_env, "production")
-      run "cd #{current_path} ; nohup bundle exec sidekiq -e #{rails_env} -C #{current_path}/config/sidekiq.yml -P #{current_path}/tmp/pids/sidekiq.pid >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
+      run "cd #{current_path} ; nohup #{fetch(:bundle_cmd, "bundle")} exec sidekiq -e #{rails_env} -C #{current_path}/config/sidekiq.yml -P #{current_path}/tmp/pids/sidekiq.pid >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
     end
 
     desc "Restart sidekiq"

data/lib/sidekiq/client.rb

@@ -19,26 +19,33 @@ module Sidekiq
       Sidekiq.redis { |x| x.smembers('queues') }
     end
 
-    # DEPRECATED
-    def self.queue_mappings
-      @queue_mappings ||= {}
-    end
-
-    # Example usage:
-    # Sidekiq::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    ##
+    # The main method used to push a job to Redis.  Accepts a number of options:
+    #
+    #   queue - the named queue to use, default 'default'
+    #   class - the worker class to call, required
+    #   args - an array of simple arguments to the perform method, must be JSON-serializable
+    #   retry - whether to retry this job if it fails, true or false, default true
+    #
+    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # symbols in 'args' will be converted to strings.
+    #
+    # Example:
+    #   Sidekiq::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #
     def self.push(item)
       raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
       raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
       raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].is_a?(Class) || !item['class'].respond_to?('get_sidekiq_options')
 
       item['retry'] = !!item['class'].get_sidekiq_options['retry']
-      queue = item['queue'] || item['class'].get_sidekiq_options['queue'] || queue_mappings[item['class'].to_s] || 'default'
+      queue = item['queue'] || item['class'].get_sidekiq_options['queue'] || 'default'
       worker_class = item['class']
       item['class'] = item['class'].to_s
 
       pushed = false
       Sidekiq.client_middleware.invoke(worker_class, item, queue) do
-        payload = MultiJson.encode(item)
+        payload = Sidekiq.dump_json(item)
         Sidekiq.redis do |conn|
           _, pushed = conn.multi do
             conn.sadd('queues', queue)

data/lib/sidekiq/fetch.rb

@@ -10,7 +10,6 @@ module Sidekiq
     include Celluloid
     include Sidekiq::Util
 
-    # Timeout for Redis#blpop.
     TIMEOUT = 1
 
     def initialize(mgr, queues)
@@ -29,13 +28,20 @@ module Sidekiq
     # a new fetch if the current fetch turned up nothing.
     def fetch
       watchdog('Fetcher#fetch died') do
-        queue = nil
-        msg = nil
-        Sidekiq.redis { |conn| queue, msg = conn.blpop(*queues_cmd) }
+        begin
+          queue = nil
+          msg = nil
+          Sidekiq.redis { |conn| queue, msg = conn.blpop(*queues_cmd) }
 
-        if msg
-          @mgr.assign!(msg, queue.gsub(/.*queue:/, ''))
-        else
+          if msg
+            @mgr.assign!(msg, queue.gsub(/.*queue:/, ''))
+          else
+            after(0) { fetch }
+          end
+        rescue => ex
+          logger.error("Error fetching message: #{ex}")
+          logger.error(ex.backtrace.first)
+          sleep(TIMEOUT)
           after(0) { fetch }
         end
       end

data/lib/sidekiq/manager.rb

@@ -19,7 +19,7 @@ module Sidekiq
     trap_exit :processor_died
 
     def initialize(options={})
-      logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.location}}"
+      logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.id}}"
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.debug { options.inspect }
       @count = options[:concurrency] || 25
@@ -110,7 +110,7 @@ module Sidekiq
           processor = @ready.pop
           @in_progress[processor.object_id] = [msg, queue]
           @busy << processor
-          processor.process!(MultiJson.decode(msg), queue)
+          processor.process!(Sidekiq.load_json(msg), queue)
         end
       end
     end

data/lib/sidekiq/middleware/client/unique_jobs.rb

@@ -1,5 +1,5 @@
-require 'multi_json'
 require 'digest'
+require 'multi_json'
 
 module Sidekiq
   module Middleware
@@ -10,7 +10,7 @@ module Sidekiq
         def call(worker_class, item, queue)
           enabled = worker_class.get_sidekiq_options['unique']
           if enabled
-            payload_hash = Digest::MD5.hexdigest(MultiJson.encode(item))
+            payload_hash = Digest::MD5.hexdigest(Sidekiq.dump_json(item))
             unique = false
 
             Sidekiq.redis do |conn|

data/lib/sidekiq/middleware/server/failure_jobs.rb

@@ -1,3 +1,5 @@
+require 'multi_json'
+
 module Sidekiq
   module Middleware
     module Server
@@ -14,8 +16,7 @@ module Sidekiq
             :worker => args[1]['class'],
             :queue => args[2]
           }
-
-          Sidekiq.redis {|conn| conn.rpush(:failed, MultiJson.encode(data)) }
+          Sidekiq.redis {|conn| conn.rpush(:failed, Sidekiq.dump_json(data)) }
           raise
         end
       end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -1,3 +1,5 @@
+require 'multi_json'
+
 require 'sidekiq/retry'
 
 module Sidekiq
@@ -44,7 +46,7 @@ module Sidekiq
             delay = DELAY.call(count)
             logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
             retry_at = Time.now.to_f + delay
-            payload = MultiJson.encode(msg)
+            payload = Sidekiq.dump_json(msg)
             Sidekiq.redis do |conn|
               conn.zadd('retry', retry_at.to_s, payload)
             end

data/lib/sidekiq/middleware/server/unique_jobs.rb

@@ -1,3 +1,5 @@
+require 'multi_json'
+
 module Sidekiq
   module Middleware
     module Server
@@ -5,7 +7,7 @@ module Sidekiq
         def call(*args)
           yield
         ensure
-          json = MultiJson.encode(args[1])
+          json = Sidekiq.dump_json(args[1])
           hash = Digest::MD5.hexdigest(json)
           Sidekiq.redis {|conn| conn.del(hash) }
         end

data/lib/sidekiq/processor.rb

@@ -1,4 +1,5 @@
 require 'celluloid'
+require 'multi_json'
 require 'sidekiq/util'
 
 require 'sidekiq/middleware/server/active_record'
@@ -53,8 +54,8 @@ module Sidekiq
       redis do |conn|
         conn.multi do
           conn.set("worker:#{self}:started", Time.now.to_s)
-          conn.set("worker:#{self}", MultiJson.encode(:queue => queue, :payload => msg,
-                                                   :run_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")))
+          hash = {:queue => queue, :payload => msg, :run_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")}
+          conn.set("worker:#{self}", Sidekiq.dump_json(hash))
         end
       end
 

data/lib/sidekiq/retry.rb

@@ -46,7 +46,7 @@ module Sidekiq
 
             messages.each do |message|
               logger.debug { "Retrying #{message}" }
-              msg = MultiJson.decode(message)
+              msg = Sidekiq.load_json(message)
               conn.rpush("queue:#{msg['queue']}", message)
             end
           end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "1.1.2"
+  VERSION = "1.1.3"
 end

data/lib/sidekiq/web.rb

@@ -52,7 +52,7 @@ module Sidekiq
           Sidekiq.redis do |conn|
             conn.smembers('workers').map do |w|
               msg = conn.get("worker:#{w}")
-              msg = MultiJson.decode(msg) if msg
+              msg = Sidekiq.load_json(msg) if msg
               [w, msg]
             end.sort { |x| x[1] ? -1 : 1 }
           end
@@ -74,7 +74,7 @@ module Sidekiq
       def retries
         Sidekiq.redis do |conn|
           results = conn.zrange('retry', 0, 25, :withscores => true)
-          results.each_slice(2).map { |msg, score| [MultiJson.decode(msg), Float(score)] }
+          results.each_slice(2).map { |msg, score| [Sidekiq.load_json(msg), Float(score)] }
         end
       end
 
@@ -89,7 +89,7 @@ module Sidekiq
       def retries_with_score(score)
         Sidekiq.redis do |conn|
           results = conn.zrangebyscore('retry', score, score)
-          results.map { |msg| MultiJson.decode(msg) }
+          results.map { |msg| Sidekiq.load_json(msg) }
         end
       end
 
@@ -124,7 +124,7 @@ module Sidekiq
     get "/queues/:name" do
       halt 404 unless params[:name]
       @name = params[:name]
-      @messages = Sidekiq.redis {|conn| conn.lrange("queue:#{@name}", 0, 10) }.map { |str| MultiJson.decode(str) }
+      @messages = Sidekiq.redis {|conn| conn.lrange("queue:#{@name}", 0, 10) }.map { |str| Sidekiq.load_json(str) }
       slim :queue
     end
 
@@ -142,7 +142,7 @@ module Sidekiq
           results = conn.zrangebyscore('retry', score, score)
           conn.zremrangebyscore('retry', score, score)
           results.map do |message|
-            msg = MultiJson.decode(message)
+            msg = Sidekiq.load_json(message)
             conn.rpush("queue:#{msg['queue']}", message)
           end
         end

data/lib/sidekiq/worker.rb

@@ -29,11 +29,6 @@ module Sidekiq
         Sidekiq::Client.push('class' => self, 'args' => args)
       end
 
-      def queue(name)
-        puts "DEPRECATED: `queue :name` is now `sidekiq_options :queue => :name`"
-        Sidekiq::Client.queue_mappings[self.name] = name.to_s
-      end
-
       ##
       # Allows customization for this type of Worker.
       # Legal options:
@@ -46,7 +41,7 @@ module Sidekiq
       end
 
       def get_sidekiq_options # :nodoc:
-        @sidekiq_options || { 'unique' => true, 'retry' => true, 'queue' => 'default' }
+        defined?(@sidekiq_options) ? @sidekiq_options : { 'unique' => true, 'retry' => true, 'queue' => 'default' }
       end
 
       def stringify_keys(hash) # :nodoc:

data/sidekiq.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency                  'redis-namespace'
   gem.add_dependency                  'connection_pool', '~> 0.9.0'
   gem.add_dependency                  'celluloid', '~> 0.10.0'
-  gem.add_dependency                  'multi_json', '~> 1'
+  gem.add_dependency                  'multi_json', '~> 1.0'
   gem.add_development_dependency      'minitest'
   gem.add_development_dependency      'sinatra'
   gem.add_development_dependency      'slim'

data/test/test_retry.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'multi_json'
 require 'sidekiq/retry'
 require 'sidekiq/middleware/server/retry_jobs'
 
@@ -81,7 +82,7 @@ class TestRetry < MiniTest::Unit::TestCase
     end
 
     it 'should poll like a bad mother...SHUT YO MOUTH' do
-      fake_msg = MultiJson.encode({ 'class' => 'Bob', 'args' => [1,2], 'queue' => 'someq' })
+      fake_msg = Sidekiq.dump_json({ 'class' => 'Bob', 'args' => [1,2], 'queue' => 'someq' })
       @redis.expect :multi, [[fake_msg], 1], []
       @redis.expect :rpush, 1, ['queue:someq', fake_msg]
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-18 00:00:00.000000000 Z
+date: 2012-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70246171142400 !ruby/object:Gem::Requirement
+  requirement: &70116788678440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246171142400
+  version_requirements: *70116788678440
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70246171141920 !ruby/object:Gem::Requirement
+  requirement: &70116788677560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246171141920
+  version_requirements: *70116788677560
 - !ruby/object:Gem::Dependency
   name: connection_pool
-  requirement: &70246171141340 !ruby/object:Gem::Requirement
+  requirement: &70116788676520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.9.0
   type: :runtime
   prerelease: false
-  version_requirements: *70246171141340
+  version_requirements: *70116788676520
 - !ruby/object:Gem::Dependency
   name: celluloid
-  requirement: &70246171140740 !ruby/object:Gem::Requirement
+  requirement: &70116788675940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,21 +54,21 @@ dependencies:
         version: 0.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *70246171140740
+  version_requirements: *70116788675940
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70246171140040 !ruby/object:Gem::Requirement
+  requirement: &70116788675040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70246171140040
+  version_requirements: *70116788675040
 - !ruby/object:Gem::Dependency
   name: minitest
-  requirement: &70246171138540 !ruby/object:Gem::Requirement
+  requirement: &70116788674260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70246171138540
+  version_requirements: *70116788674260
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70246171138080 !ruby/object:Gem::Requirement
+  requirement: &70116788673760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70246171138080
+  version_requirements: *70116788673760
 - !ruby/object:Gem::Dependency
   name: slim
-  requirement: &70246171137580 !ruby/object:Gem::Requirement
+  requirement: &70116788689600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70246171137580
+  version_requirements: *70116788689600
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70246171136620 !ruby/object:Gem::Requirement
+  requirement: &70116788689140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70246171136620
+  version_requirements: *70116788689140
 - !ruby/object:Gem::Dependency
   name: actionmailer
-  requirement: &70246171135460 !ruby/object:Gem::Requirement
+  requirement: &70116788688560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,10 +120,10 @@ dependencies:
         version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70246171135460
+  version_requirements: *70116788688560
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70246171134380 !ruby/object:Gem::Requirement
+  requirement: &70116788688060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,7 +131,7 @@ dependencies:
         version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70246171134380
+  version_requirements: *70116788688060
 description: Simple, efficient message processing for Ruby
 email:
 - mperham@gmail.com
@@ -194,7 +194,6 @@ files:
 - myapp/.gitignore
 - myapp/Capfile
 - myapp/Gemfile
-- myapp/Gemfile.lock
 - myapp/Rakefile
 - myapp/app/controllers/application_controller.rb
 - myapp/app/controllers/work_controller.rb
@@ -308,4 +307,3 @@ test_files:
 - test/test_stats.rb
 - test/test_testing.rb
 - test/test_web.rb
-has_rdoc: