sidekiq 2.8.0 → 2.9.0

data/Changes.md

@@ -1,3 +1,17 @@
+2.9.0
+-----------
+
+- Update 'sidekiq/testing' to work with any Sidekiq::Client call. It
+  also serializes the arguments as using Redis would. [#713]
+- Raise a Sidekiq::Shutdown error within workers which don't finish within the hard
+  timeout.  This is to prevent unwanted database transaction commits. [#377]
+- Lazy load Redis connection pool, you no longer need to specify
+  anything in Passenger or Unicorn's after_fork callback [#794]
+- Add optional Worker#retries_exhausted hook after max retries failed. [jkassemi, #780]
+- Fix bug in pagination link to last page [pitr, #774]
+- Upstart scripts for multiple Sidekiq instances [dariocravero, #763]
+- Use select via pipes instead of poll to catch signals [mrnugget, #761]
+
 2.8.0
 -----------
 

data/README.md

@@ -1,8 +1,11 @@
 Sidekiq
 ==============
 
-[![Build Status](https://secure.travis-ci.org/mperham/sidekiq.png)](http://travis-ci.org/mperham/sidekiq)
-[![Dependency Status](https://gemnasium.com/mperham/sidekiq.png)](https://gemnasium.com/mperham/sidekiq)
+  - [![Gem Version](https://badge.fury.io/rb/sidekiq.png)](https://rubygems.org/gems/sidekiq)
+  - [![Code Climate](https://codeclimate.com/github/mperham/sidekiq.png)](https://codeclimate.com/github/mperham/sidekiq)
+  - [![Build Status](https://travis-ci.org/mperham/sidekiq.png)](https://travis-ci.org/mperham/sidekiq)
+  - [![Coverage Status](https://coveralls.io/repos/mperham/sidekiq/badge.png?branch=master)](https://coveralls.io/r/mperham/sidekiq)
+
 
 Simple, efficient message processing for Ruby.
 
@@ -43,7 +46,10 @@ Getting Started
 -----------------
 
 See the [sidekiq home page](http://mperham.github.com/sidekiq) for the simple 4-step process.
-You can watch [Railscast #366](http://railscasts.com/episodes/366-sidekiq) to see Sidekiq in action.
+You can watch [Railscast #366](http://railscasts.com/episodes/366-sidekiq) to see Sidekiq in action.  If you do everything right, you should see this: 
+
+![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
+
 
 
 More Information

data/lib/sidekiq.rb

@@ -27,7 +27,7 @@ module Sidekiq
     :profile => false,
   }
 
-  def self.❨╯°□°❩╯︵ ┻━┻
+  def self.❨╯°□°❩╯︵┻━┻
     puts "Calm down, bro"
   end
 
@@ -68,16 +68,15 @@ module Sidekiq
 
   def self.redis(&block)
     raise ArgumentError, "requires a block" if !block
-    @redis ||= Sidekiq::RedisConnection.create
+    @redis ||= Sidekiq::RedisConnection.create(@hash || {})
     @redis.with(&block)
   end
 
   def self.redis=(hash)
+    return @redis = hash if hash.is_a?(ConnectionPool)
+
     if hash.is_a?(Hash)
-      @redis = RedisConnection.create(hash)
-      options[:namespace] ||= hash[:namespace]
-    elsif hash.is_a?(ConnectionPool)
-      @redis = hash
+      @hash = hash
     else
       raise ArgumentError, "redis= requires a Hash or ConnectionPool"
     end
@@ -115,16 +114,4 @@ module Sidekiq
     self.options[:poll_interval] = interval
   end
 
-  ##
-  # deprecated
-  def self.size(*queues)
-    return Sidekiq::Stats.new.enqueued if queues.empty?
-
-    Sidekiq.redis { |conn|
-      conn.multi {
-        queues.map { |q| conn.llen("queue:#{q}") }
-      }
-    }.inject(0) { |memo, count| memo += count }
-  end
-
 end

data/lib/sidekiq/cli.rb

@@ -1,14 +1,3 @@
-$sidekiq_signals = []
-
-# Signal handlers should do as little as humanly possible
-# and defer all work to a non-trap context.  We'll have
-# the main thread poll for signals and handle them there.
-%w(INT TERM USR1 USR2 TTIN).each do |sig|
-  trap sig do
-    $sidekiq_signals << sig
-  end
-end
-
 $stdout.sync = true
 
 require 'yaml'
@@ -20,6 +9,11 @@ require 'sidekiq'
 require 'sidekiq/util'
 
 module Sidekiq
+  # Used to raise in workers that have not finished within the
+  # hard timeout limit.  This is needed to rollback db transactions,
+  # otherwise Ruby's Thread#kill will commit.  See #377.
+  class Shutdown < RuntimeError; end
+
   class CLI
     include Util
     include Singleton
@@ -31,8 +25,6 @@ module Sidekiq
 
     def initialize
       @code = nil
-      @interrupt_mutex = Mutex.new
-      @interrupted = false
     end
 
     def parse(args=ARGV)
@@ -48,6 +40,14 @@ module Sidekiq
     end
 
     def run
+      self_read, self_write = IO.pipe
+
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+        trap sig do
+          self_write.puts(sig)
+        end
+      end
+
       logger.info "Booting Sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.id}}"
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
@@ -69,9 +69,9 @@ module Sidekiq
         end
         launcher.run
 
-        while true
-          handle_signals
-          sleep 1
+        while readable_io = IO.select([self_read])
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
         end
       rescue Interrupt
         logger.info 'Shutting down'
@@ -82,47 +82,45 @@ module Sidekiq
       end
     end
 
-    def handle_signals
-      while sig = $sidekiq_signals.shift
-        Sidekiq.logger.debug "Got #{sig} signal"
-        case sig
-        when 'INT'
-          if Sidekiq.options[:profile]
-            result = RubyProf.stop
-            printer = RubyProf::GraphHtmlPrinter.new(result)
-            File.open("profile.html", 'w') do |f|
-              printer.print(f, :min_percent => 1)
-            end
-          end
-          # Handle Ctrl-C in JRuby like MRI
-          # http://jira.codehaus.org/browse/JRUBY-4637
-          raise Interrupt
-        when 'TERM'
-          # Heroku sends TERM and then waits 10 seconds for process to exit.
-          raise Interrupt
-        when 'USR1'
-          Sidekiq.logger.info "Received USR1, no longer accepting new work"
-          launcher.manager.async.stop
-        when 'USR2'
-          if Sidekiq.options[:logfile]
-            Sidekiq.logger.info "Received USR2, reopening log file"
-            Sidekiq::Logging.initialize_logger(Sidekiq.options[:logfile])
+    private
+
+    def handle_signal(sig)
+      Sidekiq.logger.debug "Got #{sig} signal"
+      case sig
+      when 'INT'
+        if Sidekiq.options[:profile]
+          result = RubyProf.stop
+          printer = RubyProf::GraphHtmlPrinter.new(result)
+          File.open("profile.html", 'w') do |f|
+            printer.print(f, :min_percent => 1)
           end
-        when 'TTIN'
-          Thread.list.each do |thread|
-            Sidekiq.logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
-            if thread.backtrace
-              Sidekiq.logger.info thread.backtrace.join("\n")
-            else
-              Sidekiq.logger.info "<no backtrace available>"
-            end
+        end
+        # Handle Ctrl-C in JRuby like MRI
+        # http://jira.codehaus.org/browse/JRUBY-4637
+        raise Interrupt
+      when 'TERM'
+        # Heroku sends TERM and then waits 10 seconds for process to exit.
+        raise Interrupt
+      when 'USR1'
+        Sidekiq.logger.info "Received USR1, no longer accepting new work"
+        launcher.manager.async.stop
+      when 'USR2'
+        if Sidekiq.options[:logfile]
+          Sidekiq.logger.info "Received USR2, reopening log file"
+          Sidekiq::Logging.initialize_logger(Sidekiq.options[:logfile])
+        end
+      when 'TTIN'
+        Thread.list.each do |thread|
+          Sidekiq.logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+          if thread.backtrace
+            Sidekiq.logger.info thread.backtrace.join("\n")
+          else
+            Sidekiq.logger.info "<no backtrace available>"
           end
         end
       end
     end
 
-    private
-
     def load_celluloid
       raise "Celluloid cannot be required until here, or it will break Sidekiq's daemonization" if defined?(::Celluloid) && options[:daemon]
 

data/lib/sidekiq/client.rb

@@ -4,134 +4,135 @@ require 'sidekiq/middleware/chain'
 
 module Sidekiq
   class Client
+    class << self
 
-    def self.default_middleware
-      Middleware::Chain.new do |m|
+      def default_middleware
+        Middleware::Chain.new do
+        end
       end
-    end
-
-    def self.registered_workers
-      Sidekiq.redis { |x| x.smembers('workers') }
-    end
 
-    def self.registered_queues
-      Sidekiq.redis { |x| x.smembers('queues') }
-    end
+      def registered_workers
+        Sidekiq.redis { |x| x.smembers('workers') }
+      end
 
-    ##
-    # The main method used to push a job to Redis.  Accepts a number of options:
-    #
-    #   queue - the named queue to use, default 'default'
-    #   class - the worker class to call, required
-    #   args - an array of simple arguments to the perform method, must be JSON-serializable
-    #   retry - whether to retry this job if it fails, true or false, default true
-    #   backtrace - whether to save any error backtrace, default false
-    #
-    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
-    # symbols in 'args' will be converted to strings.
-    #
-    # Returns nil if not pushed to Redis or a unique Job ID if pushed.
-    #
-    # Example:
-    #   Sidekiq::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
-    #
-    def self.push(item)
-      normed = normalize_item(item)
-      normed, payload = process_single(item['class'], normed)
-
-      pushed = false
-      pushed = raw_push(normed, payload) if normed
-      pushed ? normed['jid'] : nil
-    end
+      def registered_queues
+        Sidekiq.redis { |x| x.smembers('queues') }
+      end
 
-    ##
-    # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing tens of thousands of jobs or more.  This method
-    # basically cuts down on the redis round trip latency.
-    #
-    # Takes the same arguments as Client.push except that args is expected to be
-    # an Array of Arrays.  All other keys are duplicated for each job.  Each job
-    # is run through the client middleware pipeline and each job gets its own Job ID
-    # as normal.
-    #
-    # Returns the number of jobs pushed or nil if the pushed failed.  The number of jobs
-    # pushed can be less than the number given if the middleware stopped processing for one
-    # or more jobs.
-    def self.push_bulk(items)
-      normed = normalize_item(items)
-      payloads = items['args'].map do |args|
-        _, payload = process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12)))
-        payload
-      end.compact
-
-      pushed = false
-      pushed = raw_push(normed, payloads)
-      pushed ? payloads.size : nil
-    end
+      ##
+      # The main method used to push a job to Redis.  Accepts a number of options:
+      #
+      #   queue - the named queue to use, default 'default'
+      #   class - the worker class to call, required
+      #   args - an array of simple arguments to the perform method, must be JSON-serializable
+      #   retry - whether to retry this job if it fails, true or false, default true
+      #   backtrace - whether to save any error backtrace, default false
+      #
+      # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+      # symbols in 'args' will be converted to strings.
+      #
+      # Returns nil if not pushed to Redis or a unique Job ID if pushed.
+      #
+      # Example:
+      #   Sidekiq::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+      #
+      def push(item)
+        normed = normalize_item(item)
+        payload = process_single(item['class'], normed)
+
+        pushed = false
+        pushed = raw_push([payload]) if payload
+        pushed ? payload['jid'] : nil
+      end
 
-    # Resque compatibility helpers.
-    #
-    # Example usage:
-    #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
-    #
-    # Messages are enqueued to the 'default' queue.
-    #
-    def self.enqueue(klass, *args)
-      klass.client_push('class' => klass, 'args' => args)
-    end
+      ##
+      # Push a large number of jobs to Redis.  In practice this method is only
+      # useful if you are pushing tens of thousands of jobs or more.  This method
+      # basically cuts down on the redis round trip latency.
+      #
+      # Takes the same arguments as Client.push except that args is expected to be
+      # an Array of Arrays.  All other keys are duplicated for each job.  Each job
+      # is run through the client middleware pipeline and each job gets its own Job ID
+      # as normal.
+      #
+      # Returns the number of jobs pushed or nil if the pushed failed.  The number of jobs
+      # pushed can be less than the number given if the middleware stopped processing for one
+      # or more jobs.
+      def push_bulk(items)
+        normed = normalize_item(items)
+        payloads = items['args'].map do |args|
+          raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !args.is_a?(Array)
+          process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12)))
+        end.compact
+
+        pushed = false
+        pushed = raw_push(payloads) if !payloads.empty?
+        pushed ? payloads.size : nil
+      end
 
-    # Example usage:
-    #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
-    #
-    def self.enqueue_to(queue, klass, *args)
-      klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
-    end
+      # Resque compatibility helpers.
+      #
+      # Example usage:
+      #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+      #
+      # Messages are enqueued to the 'default' queue.
+      #
+      def enqueue(klass, *args)
+        klass.client_push('class' => klass, 'args' => args)
+      end
 
-    private
+      # Example usage:
+      #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+      #
+      def enqueue_to(queue, klass, *args)
+        klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
+      end
 
-    def self.raw_push(normed, payload) # :nodoc:
-      pushed = false
-      Sidekiq.redis do |conn|
-        if normed['at'] && payload.is_a?(Array)
-          pushed = conn.zadd('schedule', payload.map {|hash| [normed['at'].to_s, hash]})
-        elsif normed['at']
-          pushed = conn.zadd('schedule', normed['at'].to_s, payload)
-        else
-          _, pushed = conn.multi do
-            conn.sadd('queues', normed['queue'])
-            conn.lpush("queue:#{normed['queue']}", payload)
+      private
+
+      def raw_push(payloads)
+        pushed = false
+        Sidekiq.redis do |conn|
+          if payloads.first['at']
+            pushed = conn.zadd('schedule', payloads.map {|hash| [hash['at'].to_s, Sidekiq.dump_json(hash)]})
+          else
+            q = payloads.first['queue']
+            to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
+            _, pushed = conn.multi do
+              conn.sadd('queues', q)
+              conn.lpush("queue:#{q}", to_push)
+            end
           end
         end
+        pushed
       end
-      pushed
-    end
 
-    def self.process_single(worker_class, item)
-      queue = item['queue']
+      def process_single(worker_class, item)
+        queue = item['queue']
 
-      Sidekiq.client_middleware.invoke(worker_class, item, queue) do
-        payload = Sidekiq.dump_json(item)
-        return item, payload
+        Sidekiq.client_middleware.invoke(worker_class, item, queue) do
+          item
+        end
       end
-    end
 
-    def self.normalize_item(item)
-      raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
-      raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
-      raise(ArgumentError, "Message args must be an Array") unless item['args'].is_a?(Array)
-      raise(ArgumentError, "Message class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
-
-      if item['class'].is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].respond_to?('get_sidekiq_options')
-        normalized_item = item['class'].get_sidekiq_options.merge(item)
-        normalized_item['class'] = normalized_item['class'].to_s
-      else
-        normalized_item = Sidekiq::Worker::ClassMethods::DEFAULT_OPTIONS.merge(item)
+      def normalize_item(item)
+        raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
+        raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
+        raise(ArgumentError, "Message args must be an Array") unless item['args'].is_a?(Array)
+        raise(ArgumentError, "Message class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
+
+        if item['class'].is_a?(Class)
+          raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].respond_to?('get_sidekiq_options')
+          normalized_item = item['class'].get_sidekiq_options.merge(item)
+          normalized_item['class'] = normalized_item['class'].to_s
+        else
+          normalized_item = Sidekiq::Worker::ClassMethods::DEFAULT_OPTIONS.merge(item)
+        end
+
+        normalized_item['jid'] = SecureRandom.hex(12)
+        normalized_item
       end
 
-      normalized_item['jid'] = SecureRandom.hex(12)
-      normalized_item
     end
-
   end
 end

data/lib/sidekiq/manager.rb

@@ -110,7 +110,7 @@ module Sidekiq
 
     def hard_shutdown_in(delay)
       after(delay) do
-        watchdog("Manager#watch_for_shutdown died") do
+        watchdog("Manager#hard_shutdown_in died") do
           # We've reached the timeout and we still have busy workers.
           # They must die but their messages shall live on.
           logger.info("Still waiting for #{@busy.size} busy workers")
@@ -138,7 +138,8 @@ module Sidekiq
 
           logger.debug { "Terminating worker threads" }
           @busy.each do |processor|
-            processor.terminate if processor.alive?
+            t = processor.bare_object.actual_work_thread
+            t.raise Shutdown if processor.alive?
           end
 
           after(0) { signal(:shutdown) }

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -14,7 +14,9 @@ module Sidekiq
       # 3. after a few days, a developer deploys a fix.  the message is
       #    reprocessed successfully.
       # 4. if 3 never happens, sidekiq will eventually give up and throw the
-      #    message away.
+      #    message away. If the worker defines a method called 'retries_exhausted',
+      #    this will be called before throwing the message away. If the
+      #    'retries_exhausted' method throws an exception, it's dropped and logged.
       #
       # A message looks like:
       #
@@ -43,7 +45,6 @@ module Sidekiq
 
         # delayed_job uses the same basic formula
         DEFAULT_MAX_RETRY_ATTEMPTS = 25
-        DELAY = proc { |count| (count ** 4) + 15 + (rand(30)*(count+1)) }
 
         def call(worker, msg, queue)
           yield
@@ -73,7 +74,7 @@ module Sidekiq
           end
 
           if count < max_retry_attempts
-            delay = DELAY.call(count)
+            delay = seconds_to_delay(count)
             logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
             retry_at = Time.now.to_f + delay
             payload = Sidekiq.dump_json(msg)
@@ -82,11 +83,20 @@ module Sidekiq
             end
           else
             # Goodbye dear message, you (re)tried your best I'm sure.
-            logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
+            retries_exhausted(worker, msg)
           end
+
           raise e
         end
 
+        def retries_exhausted(worker, msg)
+          logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
+          worker.retries_exhausted(*msg['args']) if worker.respond_to?(:retries_exhausted)
+
+        rescue Exception => e
+          handle_exception(e, "Error calling retries_exhausted")
+        end
+
         def retry_attempts_from(msg_retry, default)
           if msg_retry.is_a?(Fixnum)
             msg_retry
@@ -95,6 +105,10 @@ module Sidekiq
           end
         end
 
+        def seconds_to_delay(count)
+          (count ** 4) + 15 + (rand(30)*(count+1))
+        end
+
       end
     end
   end

data/lib/sidekiq/processor.rb

@@ -15,8 +15,6 @@ module Sidekiq
     include Util
     include Celluloid
 
-#    exclusive :process
-
     def self.default_middleware
       Middleware::Chain.new do |m|
         m.add Middleware::Server::Logging
@@ -26,6 +24,11 @@ module Sidekiq
       end
     end
 
+    # store the actual working thread so we
+    # can later kill if it necessary during
+    # hard shutdown.
+    attr_accessor :actual_work_thread
+
     def initialize(boss)
       @boss = boss
     end
@@ -34,6 +37,7 @@ module Sidekiq
       msgstr = work.message
       queue = work.queue_name
       defer do
+        @actual_work_thread = Thread.current
         begin
           msg = Sidekiq.load_json(msgstr)
           klass  = msg['class'].constantize
@@ -45,6 +49,9 @@ module Sidekiq
               worker.perform(*cloned(msg['args']))
             end
           end
+        rescue Sidekiq::Shutdown
+          # Had to force kill this job because it didn't finish
+          # within the timeout.
         rescue Exception => ex
           handle_exception(ex, msg || { :message => msgstr })
           raise
@@ -60,19 +67,19 @@ module Sidekiq
       "#<Processor #{to_s}>"
     end
 
-    def to_s
+    private
+
+    def identity
       @str ||= "#{hostname}:#{process_id}-#{Thread.current.object_id}:default"
     end
 
-    private
-
     def stats(worker, msg, queue)
       redis do |conn|
         conn.multi do
-          conn.sadd('workers', self)
-          conn.setex("worker:#{self}:started", EXPIRY, Time.now.to_s)
+          conn.sadd('workers', identity)
+          conn.setex("worker:#{identity}:started", EXPIRY, Time.now.to_s)
           hash = {:queue => queue, :payload => msg, :run_at => Time.now.to_i }
-          conn.setex("worker:#{self}", EXPIRY, Sidekiq.dump_json(hash))
+          conn.setex("worker:#{identity}", EXPIRY, Sidekiq.dump_json(hash))
         end
       end
 
@@ -89,9 +96,9 @@ module Sidekiq
       ensure
         redis do |conn|
           conn.multi do
-            conn.srem("workers", self)
-            conn.del("worker:#{self}")
-            conn.del("worker:#{self}:started")
+            conn.srem("workers", identity)
+            conn.del("worker:#{identity}")
+            conn.del("worker:#{identity}:started")
             conn.incrby("stat:processed", 1)
             conn.incrby("stat:processed:#{Time.now.utc.to_date}", 1)
           end

data/lib/sidekiq/redis_connection.rb

@@ -1,24 +1,22 @@
 require 'connection_pool'
 require 'redis'
-require 'redis/namespace'
 
 module Sidekiq
   class RedisConnection
     def self.create(options={})
       url = options[:url] || determine_redis_provider || 'redis://localhost:6379/0'
-      driver = options[:driver] || 'ruby'
       # need a connection for Fetcher and Retry
       size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 2) : 5)
-      namespace = options[:namespace]
 
       ConnectionPool.new(:timeout => 1, :size => size) do
-        build_client(url, namespace, driver)
+        build_client(url, options[:namespace], options[:driver] || 'ruby')
       end
     end
 
     def self.build_client(url, namespace, driver)
       client = Redis.connect(:url => url, :driver => driver)
       if namespace
+        require 'redis/namespace'
         Redis::Namespace.new(namespace, :redis => client)
       else
         client
@@ -28,6 +26,7 @@ module Sidekiq
 
     # Not public
     def self.determine_redis_provider
+      # REDISTOGO_URL is only support for legacy reasons
       return ENV['REDISTOGO_URL'] if ENV['REDISTOGO_URL']
       provider = ENV['REDIS_PROVIDER'] || 'REDIS_URL'
       ENV[provider]

data/lib/sidekiq/testing.rb

@@ -1,4 +1,18 @@
 module Sidekiq
+
+  class Client
+    class << self
+      alias_method :raw_push_old, :raw_push
+
+      def raw_push(payloads)
+        payloads.each do |job|
+          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+        end
+        true
+      end
+    end
+  end
+
   module Worker
     ##
     # The Sidekiq testing infrastructure overrides perform_async
@@ -56,12 +70,6 @@ module Sidekiq
     #   Then I should receive a welcome email to "foo@example.com"
     #
     module ClassMethods
-      alias_method :client_push_old, :client_push
-
-      def client_push(opts)
-        jobs << opts
-        opts.object_id
-      end
 
       # Jobs queued for this worker
       def jobs

data/lib/sidekiq/testing/inline.rb

@@ -28,9 +28,10 @@ module Sidekiq
     #
     singleton_class.class_eval do
       alias_method :raw_push_old, :raw_push
-      def raw_push(normed, payload)
-        Array(payload).each do |hash|
-          normed['class'].constantize.new.perform(*Sidekiq.load_json(hash)['args'])
+      def raw_push(payload)
+        [payload].flatten.each do |item|
+          marshalled = Sidekiq.load_json(Sidekiq.dump_json(item))
+          marshalled['class'].constantize.new.perform(*marshalled['args'])
         end
 
         true

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.8.0"
+  VERSION = "2.9.0"
 end

data/lib/sidekiq/web.rb

@@ -50,7 +50,7 @@ module Sidekiq
       end
 
       def namespace
-        Sidekiq.options[:namespace]
+        @@ns ||= Sidekiq.redis {|conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
       end
 
       def root_path

data/sidekiq.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency      'rake'
   gem.add_development_dependency      'actionmailer', '~> 3'
   gem.add_development_dependency      'activerecord', '~> 3'
+  gem.add_development_dependency      'coveralls'
 end

data/test/helper.rb

@@ -1,3 +1,8 @@
+require 'coveralls'
+Coveralls.wear! do
+  add_filter "/test/"
+end
+
 ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
 if ENV.has_key?("SIMPLECOV")
   require 'simplecov'

data/test/test_api.rb

@@ -272,7 +272,7 @@ class TestApi < MiniTest::Unit::TestCase
     it 'can enumerate workers' do
       w = Sidekiq::Workers.new
       assert_equal 0, w.size
-      w.each do |x, y|
+      w.each do
         assert false
       end
 

data/test/test_client.rb

@@ -46,7 +46,7 @@ class TestClient < MiniTest::Unit::TestCase
     end
 
     it 'pushes messages to redis' do
-      @redis.expect :lpush, 1, ['queue:foo', String]
+      @redis.expect :lpush, 1, ['queue:foo', Array]
       pushed = Sidekiq::Client.push('queue' => 'foo', 'class' => MyWorker, 'args' => [1, 2])
       assert pushed
       assert_equal 24, pushed.size
@@ -54,7 +54,7 @@ class TestClient < MiniTest::Unit::TestCase
     end
 
     it 'pushes messages to redis using a String class' do
-      @redis.expect :lpush, 1, ['queue:foo', String]
+      @redis.expect :lpush, 1, ['queue:foo', Array]
       pushed = Sidekiq::Client.push('queue' => 'foo', 'class' => 'MyWorker', 'args' => [1, 2])
       assert pushed
       assert_equal 24, pushed.size
@@ -70,28 +70,28 @@ class TestClient < MiniTest::Unit::TestCase
     end
 
     it 'handles perform_async' do
-      @redis.expect :lpush, 1, ['queue:default', String]
+      @redis.expect :lpush, 1, ['queue:default', Array]
       pushed = MyWorker.perform_async(1, 2)
       assert pushed
       @redis.verify
     end
 
     it 'handles perform_async on failure' do
-      @redis.expect :lpush, nil, ['queue:default', String]
+      @redis.expect :lpush, nil, ['queue:default', Array]
       pushed = MyWorker.perform_async(1, 2)
       refute pushed
       @redis.verify
     end
 
     it 'enqueues messages to redis' do
-      @redis.expect :lpush, 1, ['queue:default', String]
+      @redis.expect :lpush, 1, ['queue:default', Array]
       pushed = Sidekiq::Client.enqueue(MyWorker, 1, 2)
       assert pushed
       @redis.verify
     end
 
     it 'enqueues messages to redis' do
-      @redis.expect :lpush, 1, ['queue:custom_queue', String]
+      @redis.expect :lpush, 1, ['queue:custom_queue', Array]
       pushed = Sidekiq::Client.enqueue_to(:custom_queue, MyWorker, 1, 2)
       assert pushed
       @redis.verify
@@ -103,7 +103,7 @@ class TestClient < MiniTest::Unit::TestCase
     end
 
     it 'enqueues to the named queue' do
-      @redis.expect :lpush, 1, ['queue:flimflam', String]
+      @redis.expect :lpush, 1, ['queue:flimflam', Array]
       pushed = QueuedWorker.perform_async(1, 2)
       assert pushed
       @redis.verify
@@ -125,12 +125,10 @@ class TestClient < MiniTest::Unit::TestCase
       Sidekiq::Queue.new.clear
     end
     it 'can push a large set of jobs at once' do
-      a = Time.now
       count = Sidekiq::Client.push_bulk('class' => QueuedWorker, 'args' => (1..1_000).to_a.map { |x| Array(x) })
       assert_equal 1_000, count
     end
     it 'can push a large set of jobs at once using a String class' do
-      a = Time.now
       count = Sidekiq::Client.push_bulk('class' => 'QueuedWorker', 'args' => (1..1_000).to_a.map { |x| Array(x) })
       assert_equal 1_000, count
     end
@@ -178,11 +176,11 @@ class TestClient < MiniTest::Unit::TestCase
 
   describe 'item normalization' do
     it 'defaults retry to true' do
-      assert_equal true, Sidekiq::Client.normalize_item('class' => QueuedWorker, 'args' => [])['retry']
+      assert_equal true, Sidekiq::Client.send(:normalize_item, 'class' => QueuedWorker, 'args' => [])['retry']
     end
 
     it "does not normalize numeric retry's" do
-      assert_equal 2, Sidekiq::Client.normalize_item('class' => CWorker, 'args' => [])['retry']
+      assert_equal 2, Sidekiq::Client.send(:normalize_item, 'class' => CWorker, 'args' => [])['retry']
     end
   end
 end

data/test/test_retry.rb

@@ -167,6 +167,36 @@ class TestRetry < MiniTest::Unit::TestCase
       # MiniTest can't assert that a method call did NOT happen!?
       assert_raises(MockExpectationError) { @redis.verify }
     end
+
+    describe "retry exhaustion" do
+      let(:worker){ MiniTest::Mock.new }
+      let(:handler){ Sidekiq::Middleware::Server::RetryJobs.new }
+      let(:msg){ {"class"=>"Bob", "args"=>[1, 2, "foo"], "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>Time.now.utc, "retry"=>3, "retry_count"=>3} }
+
+      it 'calls worker retries_exhausted after too many retries' do
+        worker.expect(:retries_exhausted, true, [1,2,"foo"]) 
+        task_misbehaving_worker
+        worker.verify
+      end
+
+      it 'handles and logs retries_exhausted failures gracefully (drops them)' do
+        def worker.retries_exhausted(*args)
+          raise 'bam!'
+        end
+
+        e = task_misbehaving_worker
+        assert_equal e.message, "kerblammo!"
+        worker.verify
+      end
+
+      def task_misbehaving_worker
+        assert_raises RuntimeError do
+          handler.call(worker, msg, 'default') do
+            raise 'kerblammo!'
+          end
+        end
+      end
+    end
   end
 
   describe 'poller' do

data/test/test_scheduling.rb

@@ -18,20 +18,20 @@ class TestScheduling < MiniTest::Unit::TestCase
     end
 
     it 'schedules a job via interval' do
-      @redis.expect :zadd, true, ['schedule', String, String]
+      @redis.expect :zadd, true, ['schedule', Array]
       assert ScheduledWorker.perform_in(600, 'mike')
       @redis.verify
     end
 
     it 'schedules a job via timestamp' do
-      @redis.expect :zadd, true, ['schedule', String, String]
+      @redis.expect :zadd, true, ['schedule', Array]
       assert ScheduledWorker.perform_in(5.days.from_now, 'mike')
       @redis.verify
     end
 
     it 'schedules multiple jobs at once' do
       @redis.expect :zadd, true, ['schedule', Array]
-      assert Sidekiq::Client.push_bulk('class' => ScheduledWorker, 'args' => ['mike', 'mike'], 'at' => 600)
+      assert Sidekiq::Client.push_bulk('class' => ScheduledWorker, 'args' => [['mike'], ['mike']], 'at' => 600)
       @redis.verify
     end
   end

data/test/test_sidekiq.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'helper'
 
 class TestSidekiq < MiniTest::Unit::TestCase
@@ -24,4 +25,13 @@ class TestSidekiq < MiniTest::Unit::TestCase
   	end
   end
 
+  describe "❨╯°□°❩╯︵┻━┻" do
+    before { $stdout = StringIO.new }
+    after  { $stdout = STDOUT }
+
+    it "allows angry developers to express their emotional constitution and remedies it" do
+      Sidekiq.❨╯°□°❩╯︵┻━┻
+      assert_equal "Calm down, bro\n", $stdout.string
+    end
+  end
 end

data/test/test_testing.rb

@@ -53,10 +53,10 @@ class TestTesting < MiniTest::Unit::TestCase
 
     after do
       # Undo override
-      Sidekiq::Worker::ClassMethods.class_eval do
-        remove_method :client_push
-        alias_method :client_push, :client_push_old
-        remove_method :client_push_old
+      (class << Sidekiq::Client; self; end).class_eval do
+        remove_method :raw_push
+        alias_method :raw_push, :raw_push_old
+        remove_method :raw_push_old
       end
     end
 
@@ -109,6 +109,14 @@ class TestTesting < MiniTest::Unit::TestCase
         StoredWorker.drain
       end
       assert_equal 0, StoredWorker.jobs.size
+
+    end
+
+    it 'round trip serializes the job arguments' do
+      assert StoredWorker.perform_async(:mike)
+      job = StoredWorker.jobs.first
+      assert_equal "mike", job['args'].first
+      StoredWorker.clear
     end
 
     class FirstWorker

data/test/test_testing_inline.rb

@@ -81,10 +81,10 @@ class TestInline < MiniTest::Unit::TestCase
     end
 
     it 'stubs the push_bulk call when in testing mode' do
-      assert Sidekiq::Client.push_bulk({'class' => InlineWorker, 'args' => [true, true]})
+      assert Sidekiq::Client.push_bulk({'class' => InlineWorker, 'args' => [[true], [true]]})
 
       assert_raises InlineError do
-        Sidekiq::Client.push_bulk({'class' => InlineWorker, 'args' => [true, false]})
+        Sidekiq::Client.push_bulk({'class' => InlineWorker, 'args' => [[true], [false]]})
       end
     end
 

data/web/views/_paging.slim

@@ -12,4 +12,4 @@
         li
           a href="#{url}?page=#{@current_page + 1}" #{@current_page + 1}
       li class="#{'disabled' if @total_size <= @current_page * @count}"
-        a href="#{url}?page=#{(@total_size / @count).ceil + 1}" »
+        a href="#{url}?page=#{(@total_size.to_f / @count).ceil}" »

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 2.8.0
+  version: 2.9.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-04 00:00:00.000000000 Z
+date: 2013-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -187,6 +187,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Simple, efficient message processing for Ruby
 email:
 - mperham@gmail.com
@@ -204,7 +220,6 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/client
 - bin/sidekiq
 - bin/sidekiqctl
 - config.ru
@@ -258,7 +273,6 @@ files:
 - test/test_retry.rb
 - test/test_scheduling.rb
 - test/test_sidekiq.rb
-- test/test_stats.rb
 - test/test_testing.rb
 - test/test_testing_inline.rb
 - test/test_web.rb
@@ -327,7 +341,6 @@ test_files:
 - test/test_retry.rb
 - test/test_scheduling.rb
 - test/test_sidekiq.rb
-- test/test_stats.rb
 - test/test_testing.rb
 - test/test_testing_inline.rb
 - test/test_web.rb

data/bin/client

@@ -1,7 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'sidekiq'
-
-10.times do
-  Sidekiq::Client.push('class' => 'HardWorker', 'args' => ['bob', 1])
-end

data/test/test_stats.rb

@@ -1,47 +0,0 @@
-require 'helper'
-require 'sidekiq'
-require 'sidekiq/processor'
-
-class TestStats < MiniTest::Unit::TestCase
-  describe 'with redis' do
-    before do
-      @redis = Sidekiq.redis = REDIS
-      Sidekiq.redis {|c| c.flushdb }
-    end
-
-    class DumbWorker
-      include Sidekiq::Worker
-      sidekiq_options :queue => 'dumbq'
-
-      def perform(arg)
-        raise 'bang' if arg == nil
-      end
-    end
-
-    describe "info counts" do
-      before do
-        @redis.with do |conn|
-          conn.rpush 'queue:foo', '{}'
-          conn.sadd 'queues', 'foo'
-
-          3.times { conn.rpush 'queue:bar', '{}' }
-          conn.sadd 'queues', 'bar'
-
-          2.times { conn.rpush 'queue:baz', '{}' }
-          conn.sadd 'queues', 'baz'
-        end
-      end
-
-      describe "size" do
-        it "returns size of queues" do
-          assert_equal 0, Sidekiq.size("foox")
-          assert_equal 1, Sidekiq.size(:foo)
-          assert_equal 1, Sidekiq.size("foo")
-          assert_equal 4, Sidekiq.size("foo", "bar")
-          assert_equal 6, Sidekiq.size
-        end
-      end
-    end
-
-  end
-end