sidekiq 0.7.0 → 0.8.0

data/Changes.md

@@ -1,3 +1,13 @@
+0.8.0
+-----------
+
+- Remove :namespace and :server CLI options (mperham)
+- Add ExceptionNotifier support (masterkain)
+- Add capistrano support (mperham)
+- Workers now log upon start and finish (mperham)
+- Messages for terminated workers are now automatically requeued (mperham)
+- Add support for Exceptional error reporting (bensie)
+
 0.7.0
 -----------
 

data/README.md

@@ -1,4 +1,4 @@
-Sidekiq [![Build Status](https://secure.travis-ci.org/mperham/sidekiq.png)](http://travis-ci.org/mperham/sidekiq)
+Sidekiq
 ==============
 
 Simple, efficient message processing for Ruby.

data/TODO.md

@@ -1 +1,2 @@
 - resque-ui-esque web ui
+- scheduled tasks [GH-66]

data/examples/config.yml

@@ -0,0 +1,11 @@
+# Sample configuration file for Sidekiq.
+# Options here can still be overridden by cmd line args.
+#   sidekiq -C config.yml
+---
+:verbose:      false
+:namespace:    test_namespace
+:server:       127.0.0.1:1234
+:concurrency:  25
+:queues:
+  - [often, 2]
+  - [seldom, 1]

data/lib/sidekiq.rb

@@ -55,8 +55,18 @@ module Sidekiq
     @redis ||= Sidekiq::RedisConnection.create
   end
 
-  def self.redis=(r)
-    @redis = r
+  def self.redis=(hash)
+    if !hash.is_a?(Hash)
+      puts "*****************************************************
+Sidekiq.redis now takes a Hash:
+old: Sidekiq.redis = Sidekiq::RedisConnection.create(:url => 'redis://foo.com', :namespace => 'abc', :size => 12)
+new: Sidekiq.redis = { :url => 'redis://foo.com', :namespace => 'xyz', :size => 12 }
+Called from #{caller[0]}
+*****************************************************"
+      @redis = hash
+    else
+      @redis = RedisConnection.create(hash)
+    end
   end
 
   def self.client_middleware

data/lib/sidekiq/capistrano.rb

@@ -0,0 +1,29 @@
+Capistrano::Configuration.instance.load do
+  after "deploy", "sidekiq:restart"
+
+  namespace :sidekiq do
+
+    desc "Force stop sidekiq"
+    task :kill do
+      run "cd #{current_path} && kill `cat tmp/pids/sidekiq.pid` && sleep 5 && kill -9 `cat tmp/pids/sidekiq.pid`"
+    end
+
+    desc "Stop sidekiq"
+    task :stop do
+      run "cd #{current_path} && kill `cat tmp/pids/sidekiq.pid`"
+    end
+
+    desc "Start sidekiq"
+    task :start do
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path} && nohup bundle exec sidekiq -e #{rails_env} -C config/sidekiq.yml -P tmp/pids/sidekiq.pid &>> log/sidekiq.log &"
+    end
+
+    desc "Restart sidekiq"
+    task :restart do
+      stop
+      start
+    end
+
+  end
+end

data/lib/sidekiq/cli.rb

@@ -13,7 +13,6 @@ require 'yaml'
 require 'optparse'
 require 'sidekiq'
 require 'sidekiq/util'
-require 'sidekiq/redis_connection'
 require 'sidekiq/manager'
 
 module Sidekiq
@@ -42,7 +41,6 @@ module Sidekiq
     end
 
     def run
-      Sidekiq.redis ||= RedisConnection.create(:url => options[:server], :namespace => options[:namespace])
       manager = Sidekiq::Manager.new(options)
       begin
         logger.info 'Starting processing, hit Ctrl-C to stop'
@@ -113,12 +111,12 @@ module Sidekiq
           set_logger_level_to_debug
         end
 
-        o.on "-n", "--namespace NAMESPACE", "namespace worker queues are under" do |arg|
-          opts[:namespace] = arg
+        o.on "-n", "--namespace NAMESPACE", "namespace worker queues are under (DEPRECATED)" do |arg|
+          puts "Namespace support has been removed, see https://github.com/mperham/sidekiq/issues/61"
         end
 
-        o.on "-s", "--server LOCATION", "Where to find Redis" do |arg|
-          opts[:server] = arg
+        o.on "-s", "--server LOCATION", "Where to find Redis (DEPRECATED)" do |arg|
+          puts "Server location support has been removed, see https://github.com/mperham/sidekiq/issues/61"
         end
 
         o.on '-e', '--environment ENV', "Application environment" do |arg|

data/lib/sidekiq/manager.rb

@@ -49,7 +49,13 @@ module Sidekiq
       logger.info("Pausing 5 seconds to allow workers to finish...")
       after(5) do
         @busy.each(&:terminate)
-        #@busy.each(&:requeue)
+        redis.with_connection do |conn|
+          conn.multi do
+            @busy.each do |busy|
+              conn.lpush("queue:#{busy.queue}", busy.msg)
+            end
+          end
+        end
         signal(:shutdown)
       end
     end
@@ -66,6 +72,7 @@ module Sidekiq
       watchdog('sidekiq processor_done crashed!') do
         @done_callback.call(processor) if @done_callback
         @busy.delete(processor)
+        processor.msg = processor.queue = nil
         if stopped?
           processor.terminate if processor.alive?
         else
@@ -91,6 +98,8 @@ module Sidekiq
       if msg
         processor = @ready.pop
         @busy << processor
+        processor.msg = msg
+        processor.queue = queue
         processor.process!(MultiJson.decode(msg), queue)
       end
       !!msg

data/lib/sidekiq/middleware/server/exception_handler.rb

@@ -0,0 +1,38 @@
+require 'sidekiq/util'
+
+module Sidekiq
+  module Middleware
+    module Server
+      class ExceptionHandler
+        include Util
+        def call(*args)
+          yield
+        rescue => ex
+          logger.warn ex
+          logger.warn ex.backtrace.join("\n")
+          send_to_airbrake(args[1], ex) if defined?(::Airbrake)
+          send_to_exceptional(args[1], ex) if defined?(::Exceptional)
+          send_to_exception_notifier(args[1], ex) if defined?(::ExceptionNotifier)
+          raise
+        end
+
+        private
+
+        def send_to_airbrake(msg, ex)
+          ::Airbrake.notify(ex, :parameters => msg)
+        end
+
+        def send_to_exceptional(msg, ex)
+          if ::Exceptional::Config.should_send_to_api?
+            ::Exceptional.context(msg)
+            ::Exceptional::Remote.error(::Exceptional::ExceptionData.new(ex))
+          end
+        end
+
+        def send_to_exception_notifier(msg, ex)
+          ::ExceptionNotifier::Notifier.background_exception_notification(ex, :data => { :message => msg })
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/server/logging.rb

@@ -0,0 +1,27 @@
+module Sidekiq
+  module Middleware
+    module Server
+      class Logging
+        def call(*args)
+          static = "#{args[0].class.to_s} MSG-#{args[0].object_id.to_s(36)}" if logger.info?
+          start = Time.now
+          logger.info { "#{static} start" }
+          yield
+          logger.info { "#{static} done: #{elapsed(start)} sec" }
+        rescue
+          logger.info { "#{static} fail: #{elapsed(start)} sec" }
+          raise
+        end
+
+        def elapsed(start)
+          (Time.now - start).to_f.round(3)
+        end
+
+        def logger
+          Sidekiq::Util.logger
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/processor.rb

@@ -2,9 +2,10 @@ require 'celluloid'
 require 'sidekiq/util'
 
 require 'sidekiq/middleware/server/active_record'
-require 'sidekiq/middleware/server/airbrake'
+require 'sidekiq/middleware/server/exception_handler'
 require 'sidekiq/middleware/server/unique_jobs'
 require 'sidekiq/middleware/server/failure_jobs'
+require 'sidekiq/middleware/server/logging'
 
 module Sidekiq
   class Processor
@@ -17,12 +18,15 @@ module Sidekiq
 
     def self.default_middleware
       Middleware::Chain.new do |m|
-        m.add Middleware::Server::Airbrake
+        m.add Middleware::Server::ExceptionHandler
+        m.add Middleware::Server::Logging
         m.add Middleware::Server::UniqueJobs
         m.add Middleware::Server::ActiveRecord
       end
     end
 
+    attr_accessor :msg, :queue
+
     def initialize(boss)
       @boss = boss
       redis.sadd('workers', self)

data/lib/sidekiq/util.rb

@@ -1,3 +1,4 @@
+require 'time'
 require 'logger'
 
 module Sidekiq
@@ -6,10 +7,18 @@ module Sidekiq
   #
   module Util
 
+    class Pretty < Logger::Formatter
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n"
+      end
+    end
+
     def self.logger
       @logger ||= begin
-        log = Logger.new(STDERR)
+        log = Logger.new(STDOUT)
         log.level = Logger::INFO
+        log.formatter = Pretty.new
         log
       end
     end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/myapp/Capfile

@@ -0,0 +1,5 @@
+load 'deploy'
+# Uncomment if you are using Rails' asset pipeline
+    # load 'deploy/assets'
+Dir['vendor/gems/*/recipes/*.rb','vendor/plugins/*/recipes/*.rb'].each { |plugin| load(plugin) }
+load 'config/deploy' # remove this line to skip loading any of the default tasks

data/myapp/Gemfile

@@ -4,3 +4,4 @@ gem 'rails', '3.2.1'
 gem 'sqlite3'
 gem 'sidekiq', :path => '..'
 gem 'resque'
+gem 'capistrano'

data/myapp/config/deploy.rb

@@ -0,0 +1,15 @@
+require 'bundler/capistrano'
+require 'sidekiq/capistrano'
+
+set :scm, :git
+set :repository, 'git@github.com:mperham/sidekiq'
+ssh_options[:forward_agent] = true
+
+default_run_options[:pty] = true # needed to run sudo
+set :user, 'mperham'
+set :application, "myapp"
+set :deploy_via, :remote_cache
+
+role :web, "localhost"
+role :app, "localhost"
+role :db,  "localhost", :primary => true

data/myapp/config/initializers/sidekiq.rb

@@ -1,6 +1,6 @@
 Sidekiq.configure_client do |config|
-  config.redis = Sidekiq::RedisConnection.create(:namespace => 'resque', :size => 2)
+  config.redis = { :namespace => 'resque', :size => 2 }
 end
 Sidekiq.configure_server do |config|
-  config.redis = Sidekiq::RedisConnection.create(:namespace => 'resque', :size => 25)
+  config.redis = { :namespace => 'resque', :size => 25 }
 end

data/test/config.yml

@@ -1,7 +1,5 @@
 ---
 :verbose:      false
-:namespace:    test_namespace
-:server:       127.0.0.1:1234
 :environment:  xzibit
 :require:      ./test/fake_env.rb
 :pidfile:      /tmp/sidekiq-config-test.pid

data/test/test_cli.rb

@@ -70,10 +70,6 @@ class TestCli < MiniTest::Unit::TestCase
         refute Sidekiq.options[:verbose]
       end
 
-      it 'sets namespace' do
-        assert_equal "test_namespace", Sidekiq.options[:namespace]
-      end
-
       it 'sets require file' do
         assert_equal './test/fake_env.rb', Sidekiq.options[:require]
       end
@@ -110,7 +106,6 @@ class TestCli < MiniTest::Unit::TestCase
 
         @cli.parse(['sidekiq',
                     '-C', './test/config.yml',
-                    '-n', 'sweet_story_bro',
                     '-e', 'snoop',
                     '-c', '100',
                     '-r', @tmp_lib_path,
@@ -128,10 +123,6 @@ class TestCli < MiniTest::Unit::TestCase
         assert_equal 100, Sidekiq.options[:concurrency]
       end
 
-      it 'uses namespace flag' do
-        assert_equal "sweet_story_bro", Sidekiq.options[:namespace]
-      end
-
       it 'uses require file flag' do
         assert_equal @tmp_lib_path, Sidekiq.options[:require]
       end

data/test/test_client.rb

@@ -5,7 +5,7 @@ require 'sidekiq/worker'
 class TestClient < MiniTest::Unit::TestCase
   describe 'with real redis' do
     before do
-      Sidekiq.redis = Sidekiq::RedisConnection.create(:url => 'redis://localhost/sidekiq_test')
+      Sidekiq.redis = { :url => 'redis://localhost/sidekiq_test' }
       Sidekiq.redis.flushdb
     end
 
@@ -39,7 +39,7 @@ class TestClient < MiniTest::Unit::TestCase
       def @redis.setex(*); nil; end
       def @redis.expire(*); true; end
       def @redis.with_connection; yield self; end
-      Sidekiq.redis = @redis
+      Sidekiq.instance_variable_set(:@redis, @redis)
     end
 
     it 'raises ArgumentError with invalid params' do

data/test/test_extensions.rb

@@ -10,7 +10,8 @@ class TestExtensions < MiniTest::Unit::TestCase
   describe 'sidekiq extensions' do
     before do
       Sidekiq.client_middleware.entries.clear
-      Sidekiq.redis = @redis = MiniTest::Mock.new
+      Sidekiq.instance_variable_set(:@redis, MiniTest::Mock.new)
+      @redis = Sidekiq.redis
     end
 
     class MyModel < ActiveRecord::Base

data/test/test_manager.rb

@@ -8,7 +8,8 @@ require 'connection_pool'
 class TestManager < MiniTest::Unit::TestCase
   describe 'with redis' do
     before do
-      Sidekiq.redis = @redis = Sidekiq::RedisConnection.create(:url => 'redis://localhost/sidekiq_test')
+      Sidekiq.redis = { :url => 'redis://localhost/sidekiq_test' }
+      @redis = Sidekiq.redis
       @redis.flushdb
       $processed = 0
       $mutex = Mutex.new

data/test/test_middleware.rb

@@ -6,7 +6,7 @@ require 'sidekiq/processor'
 class TestMiddleware < MiniTest::Unit::TestCase
   describe 'middleware chain' do
     before do
-      Sidekiq.redis = Sidekiq::RedisConnection.create(:url => 'redis://localhost/sidekiq_test')
+      Sidekiq.redis = { :url => 'redis://localhost/sidekiq_test' }
     end
 
     class CustomMiddleware

data/test/test_processor.rb

@@ -37,7 +37,6 @@ class TestProcessor < MiniTest::Unit::TestCase
       @boss.verify
       assert_equal 0, $invokes
       assert_equal 1, $errors.size
-      assert_equal "RuntimeError", $errors[0][:error_class]
       assert_equal msg, $errors[0][:parameters]
     end
 
@@ -45,7 +44,7 @@ class TestProcessor < MiniTest::Unit::TestCase
 end
 
 class FakeAirbrake
-  def self.notify(hash)
+  def self.notify(ex, hash)
     $errors << hash
   end
 end

data/test/test_stats.rb

@@ -5,7 +5,8 @@ require 'sidekiq/processor'
 class TestStats < MiniTest::Unit::TestCase
   describe 'with redis' do
     before do
-      Sidekiq.redis = @redis = Sidekiq::RedisConnection.create(:url => 'redis://localhost/sidekiq_test')
+      Sidekiq.redis = { :url => 'redis://localhost/sidekiq_test' }
+      @redis = Sidekiq.redis
       @redis.flushdb
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-22 00:00:00.000000000 Z
+date: 2012-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70319633540020 !ruby/object:Gem::Requirement
+  requirement: &70185139523000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70319633540020
+  version_requirements: *70185139523000
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70319633539280 !ruby/object:Gem::Requirement
+  requirement: &70185139512240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70319633539280
+  version_requirements: *70185139512240
 - !ruby/object:Gem::Dependency
   name: connection_pool
-  requirement: &70319633537820 !ruby/object:Gem::Requirement
+  requirement: &70185139511020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70319633537820
+  version_requirements: *70185139511020
 - !ruby/object:Gem::Dependency
   name: celluloid
-  requirement: &70319633533720 !ruby/object:Gem::Requirement
+  requirement: &70185139509380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70319633533720
+  version_requirements: *70185139509380
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70319633624400 !ruby/object:Gem::Requirement
+  requirement: &70185139508520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70319633624400
+  version_requirements: *70185139508520
 - !ruby/object:Gem::Dependency
   name: minitest
-  requirement: &70319633623480 !ruby/object:Gem::Requirement
+  requirement: &70185139506420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70319633623480
+  version_requirements: *70185139506420
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70319633622760 !ruby/object:Gem::Requirement
+  requirement: &70185139505520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70319633622760
+  version_requirements: *70185139505520
 - !ruby/object:Gem::Dependency
   name: actionmailer
-  requirement: &70319633621420 !ruby/object:Gem::Requirement
+  requirement: &70185139503040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70319633621420
+  version_requirements: *70185139503040
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70319633620680 !ruby/object:Gem::Requirement
+  requirement: &70185139500600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70319633620680
+  version_requirements: *70185139500600
 description: Simple, efficient message processing for Ruby
 email:
 - mperham@gmail.com
@@ -120,7 +120,6 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rvmrc
-- .travis.yml
 - COMM-LICENSE
 - Changes.md
 - Gemfile
@@ -135,10 +134,12 @@ files:
 - examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb
 - examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb
 - examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb
+- examples/config.yml
 - examples/monitrc.conf
 - examples/por.rb
 - examples/sinkiq.rb
 - lib/sidekiq.rb
+- lib/sidekiq/capistrano.rb
 - lib/sidekiq/cli.rb
 - lib/sidekiq/client.rb
 - lib/sidekiq/extensions/action_mailer.rb
@@ -149,8 +150,9 @@ files:
 - lib/sidekiq/middleware/client/resque_web_compatibility.rb
 - lib/sidekiq/middleware/client/unique_jobs.rb
 - lib/sidekiq/middleware/server/active_record.rb
-- lib/sidekiq/middleware/server/airbrake.rb
+- lib/sidekiq/middleware/server/exception_handler.rb
 - lib/sidekiq/middleware/server/failure_jobs.rb
+- lib/sidekiq/middleware/server/logging.rb
 - lib/sidekiq/middleware/server/unique_jobs.rb
 - lib/sidekiq/processor.rb
 - lib/sidekiq/rails.rb
@@ -160,6 +162,7 @@ files:
 - lib/sidekiq/version.rb
 - lib/sidekiq/worker.rb
 - myapp/.gitignore
+- myapp/Capfile
 - myapp/Gemfile
 - myapp/Gemfile.lock
 - myapp/Rakefile
@@ -178,6 +181,7 @@ files:
 - myapp/config/application.rb
 - myapp/config/boot.rb
 - myapp/config/database.yml
+- myapp/config/deploy.rb
 - myapp/config/environment.rb
 - myapp/config/environments/development.rb
 - myapp/config/environments/production.rb
@@ -245,3 +249,4 @@ test_files:
 - test/test_processor.rb
 - test/test_stats.rb
 - test/test_testing.rb
+has_rdoc: 

data/.travis.yml

@@ -1,3 +0,0 @@
-language: ruby
-rvm:
-  - 1.9.3

data/lib/sidekiq/middleware/server/airbrake.rb

@@ -1,30 +0,0 @@
-require 'sidekiq/util'
-
-module Sidekiq
-  module Middleware
-    module Server
-      class Airbrake
-        include Util
-        def call(*args)
-          yield
-        rescue => ex
-          logger.warn ex
-          logger.warn ex.backtrace.join("\n")
-          send_to_airbrake(args[1], ex) if defined?(::Airbrake)
-          raise
-        end
-
-        private
-
-        def send_to_airbrake(msg, ex)
-          ::Airbrake.notify(:error_class   => ex.class.name,
-                            :error_message => "#{ex.class.name}: #{ex.message}",
-                            :parameters    => msg)
-        end
-      end
-    end
-  end
-end
-
-
-