sidekiq 0.9.1 → 0.10.0

data/.gitignore

@@ -3,3 +3,4 @@ Gemfile.lock
 *.swp
 dump.rdb
 .rbx
+coverage/

data/Changes.md

@@ -1,3 +1,12 @@
+0.10.0
+-----------
+
+- Reworked capistrano recipe to make it more fault-tolerant [#94].
+- Automatic failure retry!  Sidekiq will now save failed messages
+  and retry them, with an exponential backoff, over about 20 days.
+  Did a message fail to process?  Just deploy a bug fix in the next
+  few days and Sidekiq will retry the message eventually.
+
 0.9.1
 -----------
 

data/README.md

@@ -3,14 +3,12 @@ Sidekiq
 
 Simple, efficient message processing for Ruby.
 
-Sidekiq aims to be compatible with Resque.  It uses the exact same
+Sidekiq is compatible with Resque.  It uses the exact same
 message format as Resque so it can integrate into an existing Resque processing farm.
 You can have Sidekiq and Resque run side-by-side at the same time and
 use the Resque client to enqueue messages in Redis to be processed by Sidekiq.
 
-Sidekiq is different from Resque in how it processes messages: it
-processes many messages concurrently per process.  Resque only processes
-one message at a time per process so it is far less memory efficient.
+At the same time, Sidekiq uses multithreading so it much more memory efficient than Resque (which forks a new process for every job).
 You'll find that you might need 50 200MB resque processes to peg your CPU
 whereas one 300MB Sidekiq process will peg the same CPU and perform the
 same amount of work.  Please see [my blog post on Resque's memory
@@ -18,6 +16,8 @@ efficiency](http://blog.carbonfive.com/2011/09/16/improving-resques-memory-effic
  and how I was able to shrink a Carbon Five client's resque processing farm
 from 9 machines to 1 machine.
 
+In sum, if your jobs are well-behaved and threadsafe, Sidekiq is probably a good replacement for Resque. If your jobs are not thread-safe or they leak memory, you may want to continue using Resque, because its forking model gives you more protection.
+
 
 Requirements
 -----------------

data/Rakefile

@@ -1,7 +1,8 @@
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'test'
-  test.warning = true
+  #SO MUCH NOISE
+  #test.warning = true
   test.pattern = 'test/**/test_*.rb'
 end
 

data/TODO.md

@@ -1 +1 @@
-- reschedule failed tasks
+- Make the Web UI less ugly

data/bin/sidekiqctl

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+stage = ARGV[0]
+pidfile = ARGV[1]
+timeout = ARGV[2].to_i
+timeout = 10 if timeout == 0
+
+def done(msg)
+  puts msg
+  exit(0)
+end
+
+done 'No pidfile given' if !pidfile
+done 'Pidfile does not exist' if !File.exist?(pidfile)
+
+pid = File.read(pidfile).to_i
+done 'Invalid pidfile content' if pid == 0
+
+begin
+  Process.getpgid(pid)
+rescue Errno::ESRCH
+  done "Process doesn't exist"
+end
+
+case stage
+when 'quiet'
+  `kill -USR1 #{pid}`
+when 'stop'
+  `kill -TERM #{pid}`
+  timeout.times do
+    begin
+      Process.getpgid(pid)
+    rescue Errno::ESRCH
+      FileUtils.rm_f pidfile
+      done 'Sidekiq shut down gracefully.'
+    end
+    sleep 1
+  end
+  `kill -9 #{pid}`
+  done 'Sidekiq shut down forcefully.'
+end

data/examples/chef/cookbooks/sidekiq/README.rdoc

@@ -5,3 +5,7 @@ Sidekiq is a Redis-backed Ruby library for creating background jobs, placing tho
 = USAGE: 
 
 add require_recipe "sidekiq" to main/recipes/default.rb
+
+= NOTES:
+
+I setup a basic size for the Sidekiq workers based on the instance_type, if you need more or less workers please modify the recipe itself.  

data/examples/chef/cookbooks/sidekiq/recipes/default.rb

@@ -2,7 +2,8 @@
 # Cookbook Name:: sidekiq
 # Recipe:: default
 #
-if ['solo', 'util'].include?(node[:instance_role])
+role = node[:instance_role]
+if role == 'solo' || (role == 'util' && node[:name] =~ /sidekiq/)
 
   # for now
   worker_count = 1

data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb

@@ -140,13 +140,13 @@ APP_SHARED="${APP_DIR}/shared"
 APP_CONFIG="${APP_SHARED}/config"
 
 if [ -e "${APP_CONFIG}/${CONF_FILE}" ]; then
-  logger -t "sidekiq_${APP}" -s "Good, found a config file. Proceeding..."
+  logger -t "sidekiq_${APP}" -s "Good, found a conf file. Proceeding..."
 else
   logger -t "sidekiq_${APP}" -s "/data/${APP}/shared/config/${CONF_FILE} not found for app: ${APP}"
   exit 1
 fi
 
-WORKER_REF=`echo $CONF_FILE | sed s/.yml.conf//`
+WORKER_REF=`echo $CONF_FILE | sed s/.conf//`
 LOG_FILE="$APP_ROOT/log/$WORKER_REF.log"
 LOCK_FILE="/tmp/$WORKER_REF.monit-lock"
 PID_FILE="/var/run/engineyard/sidekiq/$APP/$WORKER_REF.pid"

data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb

@@ -2,12 +2,6 @@
 <% if @verbose %>
 :verbose: <%= @verbose %>
 <% end %>
-<% if @namespace %>
-:namespace: <%= @namespace %>
-<% end %>
-<% if @server %>
-:server: <%= @server %>
-<% end %>
 <% if @environment %>
 :environment: <%= @environment %>
 <% end %>

data/examples/por.rb

@@ -2,12 +2,12 @@ require 'sidekiq'
 
 # If your client is single-threaded, we just need a single connection in our Redis connection pool
 Sidekiq.configure_client do |config|
-  config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :size => 1, :url => 'redis://redis.host:1234/db')
+  config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :size => 1, :url => 'redis://redis.host:1234/14')
 end
 
 # Sidekiq server is multi-threaded so our Redis connection pool size defaults to concurrency (-c)
 Sidekiq.configure_server do |config|
-  config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :url => 'redis://redis.host:1234/db')
+  config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :url => 'redis://redis.host:1234/14')
 end
 
 # Start up sidekiq via

data/examples/sinkiq.rb

@@ -12,7 +12,7 @@ class SinatraWorker
   include Sidekiq::Worker
 
   def perform(msg="lulz you forgot a msg!")
-    $redis.lpush("sinkiq-example-messages", msg)      
+    $redis.lpush("sinkiq-example-messages", msg)
   end
 end
 

data/lib/sidekiq.rb

@@ -16,6 +16,7 @@ module Sidekiq
     :require => '.',
     :environment => nil,
     :timeout => 5,
+    :enable_rails_extensions => true,
   }
 
   def self.options

data/lib/sidekiq/capistrano.rb

@@ -2,18 +2,18 @@ Capistrano::Configuration.instance.load do
   before "deploy", "sidekiq:quiet"
   after "deploy", "sidekiq:restart"
 
-  _cset(:sidekiq_timeout) { 5 }
+  _cset(:sidekiq_timeout) { 10 }
 
   namespace :sidekiq do
 
     desc "Quiet sidekiq (stop accepting new work)"
     task :quiet do
-      run "cd #{current_path} && kill -USR1 `cat #{current_path}/tmp/pids/sidekiq.pid`"
+      run "cd #{current_path} && sidekiqctl quiet #{current_path}/tmp/pids/sidekiq.pid"
     end
 
     desc "Stop sidekiq"
     task :stop do
-      run "cd #{current_path} && kill `cat #{current_path}/tmp/pids/sidekiq.pid` && sleep #{fetch :sidekiq_timeout} && kill -9 `cat #{current_path}/tmp/pids/sidekiq.pid` ; rm #{current_path}/tmp/pids/sidekiq.pid"
+      run "cd #{current_path} && sidekiqctl stop #{current_path}/tmp/pids/sidekiq.pid #{fetch :sidekiq_timeout}"
     end
 
     desc "Start sidekiq"

data/lib/sidekiq/cli.rb

@@ -28,11 +28,11 @@ module Sidekiq
     include Singleton
 
     # Used for CLI testing
-    attr_accessor :code, :manager
+    attr_accessor :code
+    attr_accessor :manager
 
     def initialize
       @code = nil
-      @manager = nil
     end
 
     def parse(args=ARGV)
@@ -44,6 +44,7 @@ module Sidekiq
       options.merge!(config.merge(cli))
 
       Sidekiq::Util.logger.level = Logger::DEBUG if options[:verbose]
+      Celluloid.logger = nil
 
       validate!
       write_pid
@@ -52,14 +53,17 @@ module Sidekiq
 
     def run
       @manager = Sidekiq::Manager.new(options)
+      poller = Sidekiq::Retry::Poller.new
       begin
         logger.info 'Starting processing, hit Ctrl-C to stop'
-        manager.start!
+        @manager.start!
+        poller.poll!
         sleep
       rescue Interrupt
         logger.info 'Shutting down'
-        manager.stop!(:shutdown => true, :timeout => options[:timeout])
-        manager.wait(:shutdown)
+        poller.terminate
+        @manager.stop!(:shutdown => true, :timeout => options[:timeout])
+        @manager.wait(:shutdown)
       end
     end
 

data/lib/sidekiq/manager.rb

@@ -122,7 +122,7 @@ module Sidekiq
           @ready.size.times do
             found ||= find_work(@queues.sample)
           end
-          break logger.debug('nothing to process') unless found
+          break unless found
         end
 
         # This is the polling loop that ensures we check Redis every

data/lib/sidekiq/middleware/chain.rb

@@ -64,6 +64,10 @@ module Sidekiq
         entries.map(&:make_new)
       end
 
+      def clear
+        entries.clear
+      end
+
       def invoke(*args, &final_action)
         chain = retrieve.dup
         traverse_chain = lambda do

data/lib/sidekiq/middleware/client/unique_jobs.rb

@@ -9,13 +9,23 @@ module Sidekiq
 
         def call(item, queue)
           payload_hash = Digest::MD5.hexdigest(MultiJson.encode(item))
+          unique = false
+
           Sidekiq.redis do |conn|
-            return if conn.get(payload_hash)
-            conn.setex(payload_hash, HASH_KEY_EXPIRATION, 1)
+            conn.watch(payload_hash)
+
+            if conn.get(payload_hash)
+              conn.unwatch
+            else
+              unique = conn.multi do
+                conn.setex(payload_hash, HASH_KEY_EXPIRATION, 1)
+              end
+            end
           end
 
-          yield
+          yield if unique
         end
+
       end
     end
   end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -0,0 +1,59 @@
+require 'sidekiq/retry'
+
+module Sidekiq
+  module Middleware
+    module Server
+      ##
+      # Automatically retry jobs that fail in Sidekiq.
+      # A message looks like:
+      #
+      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'] }
+      #
+      # We'll add a bit more data to the message to support retries:
+      #
+      #  * 'queue' - the queue to use
+      #  * 'retry_count' - number of times we've retried so far.
+      #  * 'error_message' - the message from the exception
+      #  * 'error_class' - the exception class
+      #  * 'failed_at' - the first time it failed
+      #  * 'retried_at' - the last time it was retried
+      #
+      # We don't store the backtrace as that can add a lot of overhead
+      # to the message and everyone is using Airbrake, right?
+      class RetryJobs
+        include Sidekiq::Util
+        include Sidekiq::Retry
+
+        def call(worker, msg, queue)
+          yield
+        rescue => e
+          msg['queue'] = queue
+          msg['error_message'] = e.message
+          msg['error_class'] = e.class.name
+          count = if msg['retry_count']
+            msg['retried_at'] = Time.now.utc
+            msg['retry_count'] += 1
+          else
+            msg['failed_at'] = Time.now.utc
+            msg['retry_count'] = 0
+          end
+
+          if count <= MAX_COUNT
+            delay = DELAY.call(count)
+            logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+            retry_at = Time.now.to_f + delay
+            payload = MultiJson.encode(msg)
+            Sidekiq.redis do |conn|
+              conn.zadd('retry', retry_at.to_s, payload)
+            end
+          else
+            # Goodbye dear message, you (re)tried your best I'm sure.
+            logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
+          end
+          raise
+        end
+
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/server/unique_jobs.rb

@@ -5,7 +5,9 @@ module Sidekiq
         def call(*args)
           yield
         ensure
-          Sidekiq.redis {|conn| conn.del(Digest::MD5.hexdigest(MultiJson.encode(args[1]))) }
+          json = MultiJson.encode(args[1])
+          hash = Digest::MD5.hexdigest(json)
+          Sidekiq.redis {|conn| conn.del(hash) }
         end
       end
     end

data/lib/sidekiq/processor.rb

@@ -4,7 +4,7 @@ require 'sidekiq/util'
 require 'sidekiq/middleware/server/active_record'
 require 'sidekiq/middleware/server/exception_handler'
 require 'sidekiq/middleware/server/unique_jobs'
-require 'sidekiq/middleware/server/failure_jobs'
+require 'sidekiq/middleware/server/retry_jobs'
 require 'sidekiq/middleware/server/logging'
 
 module Sidekiq
@@ -21,6 +21,7 @@ module Sidekiq
         m.add Middleware::Server::ExceptionHandler
         m.add Middleware::Server::Logging
         m.add Middleware::Server::UniqueJobs
+        m.add Middleware::Server::RetryJobs
         m.add Middleware::Server::ActiveRecord
       end
     end

data/lib/sidekiq/rails.rb

@@ -1,5 +1,6 @@
 module Sidekiq
   def self.hook_rails!
+    return unless Sidekiq.options[:enable_rails_extensions]
     if defined?(ActiveRecord)
       ActiveRecord::Base.extend(Sidekiq::Extensions::ActiveRecord)
       ActiveRecord::Base.send(:include, Sidekiq::Extensions::ActiveRecord)

data/lib/sidekiq/retry.rb

@@ -0,0 +1,57 @@
+require 'sidekiq'
+require 'celluloid'
+
+module Sidekiq
+  ##
+  # Sidekiq's retry support assumes a typical development lifecycle:
+  # 0. push some code changes with a bug in it
+  # 1. bug causes message processing to fail, sidekiq's middleware captures
+  #    the message and pushes it onto a retry queue
+  # 2. sidekiq retries messages in the retry queue multiple times with
+  #    an exponential delay, the message continues to fail
+  # 3. after a few days, a developer deploys a fix.  the message is
+  #    reprocessed successfully.
+  # 4. if 3 never happens, sidekiq will eventually give up and throw the
+  #    message away.
+  module Retry
+
+    # delayed_job uses the same basic formula
+    MAX_COUNT = 25
+    DELAY = proc { |count| (count ** 4) + 15 }
+    POLL_INTERVAL = 15
+
+    ##
+    # The Poller checks Redis every N seconds for messages in the retry
+    # set have passed their retry timestamp and should be retried.  If so, it
+    # just pops the message back onto its original queue so the
+    # workers can pick it up like any other message.
+    class Poller
+      include Celluloid
+      include Sidekiq::Util
+
+      def poll
+        watchdog('retry poller thread died!') do
+
+          Sidekiq.redis do |conn|
+            # A message's "score" in Redis is the time at which it should be retried.
+            # Just check Redis for the set of messages with a timestamp before now.
+            messages = nil
+            now = Time.now.to_f.to_s
+            (messages, _) = conn.multi do
+              conn.zrangebyscore('retry', '-inf', now)
+              conn.zremrangebyscore('retry', '-inf', now)
+            end
+
+            messages.each do |message|
+              logger.debug { "Retrying #{message}" }
+              msg = MultiJson.decode(message)
+              conn.rpush("queue:#{msg['queue']}", message)
+            end
+          end
+
+          after(POLL_INTERVAL) { poll }
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "0.9.1"
+  VERSION = "0.10.0"
 end

data/lib/sidekiq/web.rb

@@ -56,6 +56,17 @@ module Sidekiq
         Sidekiq.redis { |conn| conn.get('stat:failed') } || 0
       end
 
+      def retry_count
+        Sidekiq.redis { |conn| conn.zcard('retry') }
+      end
+
+      def retries
+        Sidekiq.redis do |conn|
+          results = conn.zrange('retry', 0, 25, :withscores => true)
+          results.each_slice(2).map { |msg, score| [MultiJson.decode(msg), Float(score)] }
+        end
+      end
+
       def queues
         Sidekiq.redis do |conn|
           conn.smembers('queues').map do |q|

data/myapp/app/controllers/work_controller.rb

@@ -19,6 +19,11 @@ class WorkController < ApplicationController
     render :text => 'enqueued'
   end
 
+  def crash
+    HardWorker.perform_async('crash', 1, Time.now.to_f)
+    render :text => 'enqueued'
+  end
+
   def delayed_post
     p = Post.first
     unless p

data/myapp/app/workers/hard_worker.rb

@@ -2,6 +2,7 @@ class HardWorker
   include Sidekiq::Worker
 
   def perform(name, count, salt)
+    raise name if name == 'crash'
     print "#{Time.now}\n"
     sleep count
   end

data/myapp/config/routes.rb

@@ -6,4 +6,5 @@ Myapp::Application.routes.draw do
   get "work/email" => "work#email"
   get "work/post" => "work#delayed_post"
   get "work/long" => "work#long"
+  get "work/crash" => "work#crash"
 end

data/sidekiq.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.description   = gem.summary = "Simple, efficient message processing for Ruby"
   gem.homepage      = "http://mperham.github.com/sidekiq"
 
-  gem.executables   = ['sidekiq']
+  gem.executables   = ['sidekiq', 'sidekiqctl']
   gem.files         = `git ls-files`.split("\n")
   gem.test_files    = `git ls-files -- test/*`.split("\n")
   gem.name          = "sidekiq"
@@ -15,13 +15,13 @@ Gem::Specification.new do |gem|
   gem.version       = Sidekiq::VERSION
   gem.add_dependency                  'redis'
   gem.add_dependency                  'redis-namespace'
-  gem.add_dependency                  'connection_pool', '>= 0.9.0'
+  gem.add_dependency                  'connection_pool', '~> 0.9.0'
   gem.add_dependency                  'celluloid'
   gem.add_dependency                  'multi_json'
   gem.add_development_dependency      'minitest'
   gem.add_development_dependency      'sinatra'
   gem.add_development_dependency      'slim'
   gem.add_development_dependency      'rake'
-  gem.add_development_dependency      'actionmailer'
-  gem.add_development_dependency      'activerecord'
+  gem.add_development_dependency      'actionmailer', '~> 3'
+  gem.add_development_dependency      'activerecord', '~> 3'
 end

data/test/helper.rb

@@ -1,5 +1,4 @@
-$TESTING = true
-if false
+if ENV.has_key?("SIMPLECOV")
   require 'simplecov'
   SimpleCov.start
 end
@@ -13,4 +12,4 @@ require 'sidekiq/util'
 Sidekiq::Util.logger.level = Logger::ERROR
 
 require 'sidekiq/redis_connection'
-REDIS = Sidekiq::RedisConnection.create(:url => 'redis://localhost/sidekiq_test')
+REDIS = Sidekiq::RedisConnection.create(:url => "redis://localhost/15")

data/test/test_client.rb

@@ -28,17 +28,19 @@ class TestClient < MiniTest::Unit::TestCase
   describe 'with mock redis' do
     before do
       @redis = MiniTest::Mock.new
-      def @redis.multi; yield; end
+      def @redis.multi; yield if block_given?; end
       def @redis.set(*); true; end
       def @redis.sadd(*); true; end
       def @redis.srem(*); true; end
       def @redis.get(*); nil; end
       def @redis.del(*); nil; end
       def @redis.incrby(*); nil; end
-      def @redis.setex(*); nil; end
+      def @redis.setex(*); true; end
       def @redis.expire(*); true; end
+      def @redis.watch(*); true; end
       def @redis.with_connection; yield self; end
       def @redis.with; yield self; end
+      def @redis.exec; true; end
       Sidekiq.instance_variable_set(:@redis, @redis)
     end
 

data/test/test_extensions.rb

@@ -28,10 +28,6 @@ class TestExtensions < MiniTest::Unit::TestCase
       assert_equal 1, Sidekiq.redis.llen('queue:default')
     end
 
-    it 'allows delayed exection of ActiveRecord instance methods' do
-      skip('requires a database')
-    end
-
     class UserMailer < ActionMailer::Base
       def greetings(a, b)
         raise "Should not be called!"
@@ -45,6 +41,28 @@ class TestExtensions < MiniTest::Unit::TestCase
       assert_equal ['default'], Sidekiq::Client.registered_queues
       assert_equal 1, Sidekiq.redis.llen('queue:default')
     end
+  end
 
+  describe 'sidekiq rails extensions configuration' do
+    before do
+      @options = Sidekiq.options
+    end
+
+    after do
+      Sidekiq.options = @options
+    end
+
+    it 'should set enable_rails_extensions option to true by default' do
+      assert Sidekiq.options[:enable_rails_extensions]
+    end
+
+    it 'should extend ActiveRecord and ActiveMailer if enable_rails_extensions is true' do
+      assert Sidekiq.hook_rails!
+    end
+
+    it 'should not extend ActiveRecord and ActiveMailer if enable_rails_extensions is false' do
+      Sidekiq.options = { :enable_rails_extensions => false }
+      refute Sidekiq.hook_rails!
+    end
   end
 end

data/test/test_middleware.rb

@@ -6,9 +6,22 @@ require 'sidekiq/processor'
 class TestMiddleware < MiniTest::Unit::TestCase
   describe 'middleware chain' do
     before do
+      $errors = []
       Sidekiq.redis = REDIS
     end
 
+    it 'handles errors' do
+      handler = Sidekiq::Middleware::Server::ExceptionHandler.new
+
+      assert_raises ArgumentError do
+        handler.call('', { :a => 1 }, 'default') do
+          raise ArgumentError
+        end
+      end
+      assert_equal 1, $errors.size
+      assert_equal({ :a => 1 }, $errors[0][:parameters])
+    end
+
     class CustomMiddleware
       def initialize(name, recorder)
         @name = name
@@ -42,10 +55,6 @@ class TestMiddleware < MiniTest::Unit::TestCase
     end
 
     it 'executes middleware in the proper order' do
-      Sidekiq::Middleware::Server::UniqueJobs.class_eval do
-        def call(*args); yield; end
-      end
-
       recorder = []
       msg = { 'class' => CustomWorker.to_s, 'args' => [recorder] }
 
@@ -74,3 +83,10 @@ class TestMiddleware < MiniTest::Unit::TestCase
     end
   end
 end
+
+class FakeAirbrake
+  def self.notify(ex, hash)
+    $errors << hash
+  end
+end
+Airbrake = FakeAirbrake

data/test/test_processor.rb

@@ -8,6 +8,7 @@ class TestProcessor < MiniTest::Unit::TestCase
       $errors = []
       @boss = MiniTest::Mock.new
       Celluloid.logger = nil
+      Sidekiq.redis = REDIS
     end
 
     class MockWorker
@@ -27,26 +28,5 @@ class TestProcessor < MiniTest::Unit::TestCase
       assert_equal 1, $invokes
       assert_equal 0, $errors.size
     end
-
-    it 'handles exceptions' do
-      msg = { 'class' => MockWorker.to_s, 'args' => ['boom'] }
-      processor = ::Sidekiq::Processor.new(@boss)
-      assert_raises RuntimeError do
-        processor.process(msg, 'default')
-      end
-      @boss.verify
-      assert_equal 0, $invokes
-      assert_equal 1, $errors.size
-      assert_equal msg, $errors[0][:parameters]
-    end
-
   end
 end
-
-class FakeAirbrake
-  def self.notify(ex, hash)
-    $errors << hash
-  end
-end
-Airbrake = FakeAirbrake
-

data/test/test_retry.rb

@@ -0,0 +1,83 @@
+require 'helper'
+require 'sidekiq/retry'
+require 'sidekiq/middleware/server/retry_jobs'
+
+class TestRetry < MiniTest::Unit::TestCase
+  describe 'middleware' do
+    before do
+      @redis = MiniTest::Mock.new
+      # Ugh, this is terrible.
+      Sidekiq.instance_variable_set(:@redis, @redis)
+
+      def @redis.with; yield self; end
+    end
+
+    it 'handles a new failed message' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'] }
+      handler = Sidekiq::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call('', msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'default', msg["queue"]
+      assert_equal 'kerblammo!', msg["error_message"]
+      assert_equal 'RuntimeError', msg["error_class"]
+      assert_equal 0, msg["retry_count"]
+      assert msg["failed_at"]
+      @redis.verify
+    end
+
+    it 'handles a recurring failed message' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      now = Time.now.utc
+      msg = {"class"=>"Bob", "args"=>[1, 2, "foo"], "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry_count"=>10}
+      handler = Sidekiq::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call('', msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'default', msg["queue"]
+      assert_equal 'kerblammo!', msg["error_message"]
+      assert_equal 'RuntimeError', msg["error_class"]
+      assert_equal 11, msg["retry_count"]
+      assert msg["failed_at"]
+      @redis.verify
+    end
+
+    it 'throws away old messages after too many retries' do
+      now = Time.now.utc
+      msg = {"class"=>"Bob", "args"=>[1, 2, "foo"], "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry_count"=>25}
+      handler = Sidekiq::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call('', msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      @redis.verify
+    end
+  end
+
+  describe 'poller' do
+    before do
+      @redis = MiniTest::Mock.new
+      Sidekiq.instance_variable_set(:@redis, @redis)
+
+      def @redis.with; yield self; end
+    end
+
+    it 'should poll like a bad mother...SHUT YO MOUTH' do
+      fake_msg = MultiJson.encode({ 'class' => 'Bob', 'args' => [1,2], 'queue' => 'someq' })
+      @redis.expect :multi, [[fake_msg], 1], []
+      @redis.expect :rpush, 1, ['queue:someq', fake_msg]
+
+      inst = Sidekiq::Retry::Poller.new
+      inst.poll
+
+      @redis.verify
+    end
+  end
+
+end

data/web/views/index.slim

@@ -4,6 +4,7 @@
   p Processed: #{processed}
   p Failed: #{failed}
   p Workers: #{workers.size}
+  p Retries Pending: #{retry_count}
 
 .tabbable
   ul.nav.nav-tabs
@@ -11,6 +12,8 @@
       a href="#workers" data-toggle="tab" Workers
     li
       a href="#queues" data-toggle="tab" Queues
+    li
+      a href="#retries" data-toggle="tab" Retries
   .tab-content
     #workers.tab-pane.active
       table class="table table-striped table-bordered"
@@ -40,3 +43,20 @@
               a href="queues/#{queue}" #{queue}
             td= size
 
+    #retries.tab-pane
+      table class="table table-striped table-bordered"
+        tr
+          th Next Retry
+          th Retry Count
+          th Queue
+          th Worker
+          th Args
+        - retries.each do |(msg, score)|
+          tr
+            td= Time.at(score)
+            td= msg['retry_count']
+            td
+              a href="queues/#{msg['queue']}" #{msg['queue']}
+            td= msg['class']
+            td= msg['args'].inspect[0..100]
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.10.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-17 00:00:00.000000000 Z
+date: 2012-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70238239105760 !ruby/object:Gem::Requirement
+  requirement: &70249655983600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70238239105760
+  version_requirements: *70249655983600
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70238239105320 !ruby/object:Gem::Requirement
+  requirement: &70249655982380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,21 +32,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70238239105320
+  version_requirements: *70249655982380
 - !ruby/object:Gem::Dependency
   name: connection_pool
-  requirement: &70238239104820 !ruby/object:Gem::Requirement
+  requirement: &70249655981700 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.9.0
   type: :runtime
   prerelease: false
-  version_requirements: *70238239104820
+  version_requirements: *70249655981700
 - !ruby/object:Gem::Dependency
   name: celluloid
-  requirement: &70238239104400 !ruby/object:Gem::Requirement
+  requirement: &70249655981280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70238239104400
+  version_requirements: *70249655981280
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70238239103940 !ruby/object:Gem::Requirement
+  requirement: &70249655980820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70238239103940
+  version_requirements: *70249655980820
 - !ruby/object:Gem::Dependency
   name: minitest
-  requirement: &70238239103520 !ruby/object:Gem::Requirement
+  requirement: &70249655980400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70238239103520
+  version_requirements: *70249655980400
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70238239103100 !ruby/object:Gem::Requirement
+  requirement: &70249656007360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70238239103100
+  version_requirements: *70249656007360
 - !ruby/object:Gem::Dependency
   name: slim
-  requirement: &70238239102680 !ruby/object:Gem::Requirement
+  requirement: &70249656006940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70238239102680
+  version_requirements: *70249656006940
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70238239102260 !ruby/object:Gem::Requirement
+  requirement: &70249656006520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,34 +109,35 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70238239102260
+  version_requirements: *70249656006520
 - !ruby/object:Gem::Dependency
   name: actionmailer
-  requirement: &70238239101840 !ruby/object:Gem::Requirement
+  requirement: &70249656006020 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70238239101840
+  version_requirements: *70249656006020
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70238239101420 !ruby/object:Gem::Requirement
+  requirement: &70249656005520 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70238239101420
+  version_requirements: *70249656005520
 description: Simple, efficient message processing for Ruby
 email:
 - mperham@gmail.com
 executables:
 - sidekiq
+- sidekiqctl
 extensions: []
 extra_rdoc_files: []
 files:
@@ -151,6 +152,7 @@ files:
 - TODO.md
 - bin/client
 - bin/sidekiq
+- bin/sidekiqctl
 - config.ru
 - examples/chef/cookbooks/sidekiq/README.rdoc
 - examples/chef/cookbooks/sidekiq/recipes/default.rb
@@ -162,6 +164,7 @@ files:
 - examples/por.rb
 - examples/scheduling.rb
 - examples/sinkiq.rb
+- examples/web-ui.png
 - lib/sidekiq.rb
 - lib/sidekiq/capistrano.rb
 - lib/sidekiq/cli.rb
@@ -176,10 +179,12 @@ files:
 - lib/sidekiq/middleware/server/exception_handler.rb
 - lib/sidekiq/middleware/server/failure_jobs.rb
 - lib/sidekiq/middleware/server/logging.rb
+- lib/sidekiq/middleware/server/retry_jobs.rb
 - lib/sidekiq/middleware/server/unique_jobs.rb
 - lib/sidekiq/processor.rb
 - lib/sidekiq/rails.rb
 - lib/sidekiq/redis_connection.rb
+- lib/sidekiq/retry.rb
 - lib/sidekiq/testing.rb
 - lib/sidekiq/util.rb
 - lib/sidekiq/version.rb
@@ -235,6 +240,7 @@ files:
 - test/test_manager.rb
 - test/test_middleware.rb
 - test/test_processor.rb
+- test/test_retry.rb
 - test/test_stats.rb
 - test/test_testing.rb
 - test/test_web.rb
@@ -295,6 +301,7 @@ test_files:
 - test/test_manager.rb
 - test/test_middleware.rb
 - test/test_processor.rb
+- test/test_retry.rb
 - test/test_stats.rb
 - test/test_testing.rb
 - test/test_web.rb