sidekiq 0.8.0 → 0.9.0

data/Changes.md

@@ -1,3 +1,14 @@
+0.9.0
+-----------
+
+- Add -t option to configure the TERM shutdown timeout
+- TERM shutdown timeout is now configurable, defaults to 5 seconds.
+- USR1 signal now stops Sidekiq from accepting new work,
+  capistrano sends USR1 at start of deploy and TERM at end of deploy
+  giving workers the maximum amount of time to finish.
+- New Sidekiq::Web rack application available
+- Updated Sidekiq.redis API
+
 0.8.0
 -----------
 

data/Gemfile

@@ -1,4 +1,10 @@
 source 'http://rubygems.org'
 gemspec
 
-gem 'simplecov', :require => false, :group => :test
+gem 'slim'
+gem 'sprockets'
+gem 'sass'
+
+group :test do
+  gem 'simplecov', :require => false
+end

data/LICENSE

@@ -12,5 +12,11 @@ You may choose to license Sidekiq in one of two ways:
   $50 to my Pledgie fund at http://www.pledgie.com/campaigns/16623 to use
   this license.
 
-This commercial license is just meant to help support my OSS work.  If
+The commercial license is just meant to help support my OSS work.  If
 you do buy the commercial licensing option, thank you for your support!
+
+The commercial license is meant to be per-site.  Consulting shops must buy
+one license per client engagement which uses Sidekiq.  For instance,
+GitHub could license Sidekiq for $50 for their own internal use but they
+must pay $50 per GitHub::FI install because FI is a separate installation
+from the GitHub website managed by the FI customer.

data/README.md

@@ -42,6 +42,8 @@ More Information
 -----------------
 
 Please see the [sidekiq wiki](https://github.com/mperham/sidekiq/wiki) for more information.
+[#sidekiq on irc.freenode.net](irc://irc.freenode.net/#sidekiq) is dedicated to this project,
+but bug reports or feature requests suggestions should still go through [issues on Github](https://github.com/mperham/sidekiq/issues).
 
 
 License

data/TODO.md

@@ -1,2 +1 @@
-- resque-ui-esque web ui
-- scheduled tasks [GH-66]
+- reschedule failed tasks

data/bin/sidekiq

@@ -10,7 +10,7 @@ rescue LoadError
 end
 
 begin
-  cli = Sidekiq::CLI.new
+  cli = Sidekiq::CLI.instance
   cli.parse
   cli.run
 rescue => e

data/config.ru

@@ -0,0 +1,8 @@
+require 'sidekiq'
+
+Sidekiq.configure_client do |config|
+  config.redis = { :size => 1 }
+end
+
+require 'sidekiq/web'
+run Sidekiq::Web

data/examples/config.yml

@@ -3,8 +3,6 @@
 #   sidekiq -C config.yml
 ---
 :verbose:      false
-:namespace:    test_namespace
-:server:       127.0.0.1:1234
 :concurrency:  25
 :queues:
   - [often, 2]

data/examples/scheduling.rb

@@ -0,0 +1,37 @@
+# Sidekiq defers scheduling to other, better suited gems.
+# If you want to run a job regularly, here's an example
+# of using the 'whenever' gem to push jobs to Sidekiq
+# regularly.
+
+class MyWorker
+  include Sidekiq::Worker
+
+  def perform(count)
+    puts "Job ##{count}: Late night, so tired..."
+  end
+
+  def self.late_night_work
+    10.times do |x|
+      perform_async(x)
+    end
+  end
+end
+
+# Kick off a bunch of jobs early in the morning
+every 1.day, :at => '4:30 am' do
+  runner "MyWorker.late_night_work"
+end
+
+
+class HourlyWorker
+  include Sidekiq::Worker
+
+  def perform
+    cleanup_database
+    format_hard_drive
+  end
+end
+
+every :hour do # Many shortcuts available: :hour, :day, :month, :year, :reboot
+  runner "HourlyWorker.perform_async"
+end

data/lib/sidekiq.rb

@@ -1,11 +1,11 @@
 require 'sidekiq/version'
 require 'sidekiq/client'
 require 'sidekiq/worker'
-require 'sidekiq/rails' if defined?(::Rails)
+require 'sidekiq/rails'
 require 'sidekiq/redis_connection'
 
-require 'sidekiq/extensions/action_mailer' if defined?(::ActionMailer)
-require 'sidekiq/extensions/active_record' if defined?(::ActiveRecord)
+require 'sidekiq/extensions/action_mailer'
+require 'sidekiq/extensions/active_record'
 
 module Sidekiq
 
@@ -14,6 +14,7 @@ module Sidekiq
     :concurrency => 25,
     :require => '.',
     :environment => nil,
+    :timeout => 5,
   }
 
   def self.options
@@ -28,7 +29,7 @@ module Sidekiq
   # Configuration for Sidekiq server, use like:
   #
   #   Sidekiq.configure_server do |config|
-  #     config.redis = Sidekiq::RedisConnection.create(:namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/mydb')
+  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/mydb' }
   #     config.server_middleware do |chain|
   #       chain.add MyServerHook
   #     end
@@ -41,7 +42,7 @@ module Sidekiq
   # Configuration for Sidekiq client, use like:
   #
   #   Sidekiq.configure_client do |config|
-  #     config.redis = Sidekiq::RedisConnection.create(:namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/mydb')
+  #     config.redis = { :namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/mydb' }
   #   end
   def self.configure_client
     yield self unless server?
@@ -51,21 +52,34 @@ module Sidekiq
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis
+  def self.redis(&block)
     @redis ||= Sidekiq::RedisConnection.create
+    if block_given?
+      @redis.with(&block)
+    else
+      Sidekiq::Util.logger.info "*****************************************************
+Sidekiq.redis now takes a block:
+
+  Sidekiq.redis { |connection| connection.smembers('myset') }
+
+Please update your code accordingly.
+Called from #{caller[0]}
+*****************************************************"
+      @redis
+    end
   end
 
   def self.redis=(hash)
-    if !hash.is_a?(Hash)
-      puts "*****************************************************
+    @redis = if hash.is_a?(Hash)
+      RedisConnection.create(hash)
+    else
+      Sidekiq::Util.logger.info "*****************************************************
 Sidekiq.redis now takes a Hash:
 old: Sidekiq.redis = Sidekiq::RedisConnection.create(:url => 'redis://foo.com', :namespace => 'abc', :size => 12)
 new: Sidekiq.redis = { :url => 'redis://foo.com', :namespace => 'xyz', :size => 12 }
 Called from #{caller[0]}
 *****************************************************"
-      @redis = hash
-    else
-      @redis = RedisConnection.create(hash)
+      hash
     end
   end
 

data/lib/sidekiq/capistrano.rb

@@ -1,22 +1,25 @@
 Capistrano::Configuration.instance.load do
+  before "deploy", "sidekiq:quiet"
   after "deploy", "sidekiq:restart"
 
+  _cset(:sidekiq_timeout) { 5 }
+
   namespace :sidekiq do
 
-    desc "Force stop sidekiq"
-    task :kill do
-      run "cd #{current_path} && kill `cat tmp/pids/sidekiq.pid` && sleep 5 && kill -9 `cat tmp/pids/sidekiq.pid`"
+    desc "Quiet sidekiq (stop accepting new work)"
+    task :quiet do
+      run "cd #{current_path} && kill -USR1 `cat #{current_path}/tmp/pids/sidekiq.pid`"
     end
 
     desc "Stop sidekiq"
     task :stop do
-      run "cd #{current_path} && kill `cat tmp/pids/sidekiq.pid`"
+      run "cd #{current_path} && kill `cat #{current_path}/tmp/pids/sidekiq.pid` && sleep #{fetch :sidekiq_timeout} && kill -9 `cat #{current_path}/tmp/pids/sidekiq.pid` ; rm #{current_path}/tmp/pids/sidekiq.pid"
     end
 
     desc "Start sidekiq"
     task :start do
       rails_env = fetch(:rails_env, "production")
-      run "cd #{current_path} && nohup bundle exec sidekiq -e #{rails_env} -C config/sidekiq.yml -P tmp/pids/sidekiq.pid &>> log/sidekiq.log &"
+      run "cd #{current_path} && nohup bundle exec sidekiq -e #{rails_env} -C #{current_path}/config/sidekiq.yml -P #{current_path}/tmp/pids/sidekiq.pid >> #{current_path}/log/sidekiq.log 2>&1 &"
     end
 
     desc "Restart sidekiq"

data/lib/sidekiq/cli.rb

@@ -9,7 +9,14 @@ trap 'TERM' do
   Thread.main.raise Interrupt
 end
 
+trap 'USR1' do
+  Sidekiq::Util.logger.info "Received USR1, no longer accepting new work"
+  mgr = Sidekiq::CLI.instance.manager
+  mgr.stop! if mgr
+end
+
 require 'yaml'
+require 'singleton'
 require 'optparse'
 require 'sidekiq'
 require 'sidekiq/util'
@@ -18,40 +25,40 @@ require 'sidekiq/manager'
 module Sidekiq
   class CLI
     include Util
+    include Singleton
 
     # Used for CLI testing
-    attr_accessor :code
+    attr_accessor :code, :manager
 
     def initialize
       @code = nil
+      @manager = nil
     end
 
     def parse(args=ARGV)
+      @code = nil
       Sidekiq::Util.logger
 
       cli = parse_options(args)
       config = parse_config(cli)
       options.merge!(config.merge(cli))
 
-      set_logger_level_to_debug if options[:verbose]
+      Sidekiq::Util.logger.level = Logger::DEBUG if options[:verbose]
 
-      write_pid
       validate!
+      write_pid
       boot_system
     end
 
     def run
-      manager = Sidekiq::Manager.new(options)
+      @manager = Sidekiq::Manager.new(options)
       begin
         logger.info 'Starting processing, hit Ctrl-C to stop'
         manager.start!
-        # HACK need to determine how to pause main thread while
-        # waiting for signals.
         sleep
       rescue Interrupt
-        # TODO Need clean shutdown support from Celluloid
         logger.info 'Shutting down'
-        manager.stop!
+        manager.stop!(:shutdown => true, :timeout => options[:timeout])
         manager.wait(:shutdown)
       end
     end
@@ -108,21 +115,17 @@ module Sidekiq
         end
 
         o.on "-v", "--verbose", "Print more verbose output" do
-          set_logger_level_to_debug
-        end
-
-        o.on "-n", "--namespace NAMESPACE", "namespace worker queues are under (DEPRECATED)" do |arg|
-          puts "Namespace support has been removed, see https://github.com/mperham/sidekiq/issues/61"
-        end
-
-        o.on "-s", "--server LOCATION", "Where to find Redis (DEPRECATED)" do |arg|
-          puts "Server location support has been removed, see https://github.com/mperham/sidekiq/issues/61"
+          Sidekiq::Util.logger.level = Logger::DEBUG
         end
 
         o.on '-e', '--environment ENV', "Application environment" do |arg|
           opts[:environment] = arg
         end
 
+        o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
+          opts[:timeout] = arg.to_i
+        end
+
         o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
           opts[:require] = arg
         end
@@ -173,9 +176,5 @@ module Sidekiq
       end
     end
 
-    def set_logger_level_to_debug
-      Sidekiq::Util.logger.level = Logger::DEBUG
-    end
-
   end
 end

data/lib/sidekiq/client.rb

@@ -1,9 +1,7 @@
 require 'multi_json'
 require 'redis'
 
-require 'sidekiq/redis_connection'
 require 'sidekiq/middleware/chain'
-require 'sidekiq/middleware/client/resque_web_compatibility'
 require 'sidekiq/middleware/client/unique_jobs'
 
 module Sidekiq
@@ -16,16 +14,15 @@ module Sidekiq
     def self.default_middleware
       Middleware::Chain.new do |m|
         m.add Middleware::Client::UniqueJobs
-        m.add Middleware::Client::ResqueWebCompatibility
       end
     end
 
     def self.registered_workers
-      Sidekiq.redis.smembers('workers')
+      Sidekiq.redis { |x| x.smembers('workers') }
     end
 
     def self.registered_queues
-      Sidekiq.redis.smembers('queues')
+      Sidekiq.redis { |x| x.smembers('queues') }
     end
 
     def self.queue_mappings
@@ -44,7 +41,13 @@ module Sidekiq
 
       pushed = false
       Sidekiq.client_middleware.invoke(item, queue) do
-        Sidekiq.redis.rpush("queue:#{queue}", MultiJson.encode(item))
+        payload = MultiJson.encode(item)
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.sadd('queues', queue)
+            conn.rpush("queue:#{queue}", payload)
+          end
+        end
         pushed = true
       end
       pushed

data/lib/sidekiq/extensions/action_mailer.rb

@@ -22,6 +22,5 @@ module Sidekiq
       end
     end
 
-    ::ActionMailer::Base.extend(ActionMailer)
   end
-end if defined?(::ActionMailer)
+end

data/lib/sidekiq/extensions/active_record.rb

@@ -23,7 +23,5 @@ module Sidekiq
       end
     end
 
-    ::ActiveRecord::Base.extend(ActiveRecord)
-    ::ActiveRecord::Base.send(:include, ActiveRecord)
   end
-end if defined?(::ActiveRecord)
+end

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -13,7 +13,7 @@ module Sidekiq
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
         obj = [@target, name, args]
-        ::Sidekiq::Client.push('class' => @performable.name, 'args' => [::YAML.dump(obj)])
+        @performable.perform_async(::YAML.dump(obj))
       end
     end
 

data/lib/sidekiq/manager.rb

@@ -4,6 +4,7 @@ require 'multi_json'
 
 require 'sidekiq/util'
 require 'sidekiq/processor'
+require 'connection_pool/version'
 
 module Sidekiq
 
@@ -19,7 +20,8 @@ module Sidekiq
     trap_exit :processor_died
 
     def initialize(options={})
-      logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis.client.location}"
+      logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.location}}"
+      logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.debug { options.inspect }
       @count = options[:concurrency] || 25
       @queues = options[:queues]
@@ -30,33 +32,33 @@ module Sidekiq
       @ready = @count.times.map { Processor.new_link(current_actor) }
     end
 
-    def stop
+    def stop(options={})
+      shutdown = options[:shutdown]
+      timeout = options[:timeout]
+
       @done = true
-      @ready.each(&:terminate)
+      @ready.each { |x| x.terminate if x.alive? }
       @ready.clear
 
-      redis.with_connection do |conn|
+      redis do |conn|
         workers = conn.smembers('workers')
         workers.each do |name|
           conn.srem('workers', name) if name =~ /:#{process_id}-/
         end
       end
 
-      if @busy.empty?
-        return signal(:shutdown)
-      end
+      if shutdown
+        if @busy.empty?
+          # after(0) needed to avoid deadlock in Celluoid after USR1 + TERM
+          return after(0) { signal(:shutdown) }
+        else
+          logger.info { "Pausing #{timeout} seconds to allow workers to finish..." }
+        end
 
-      logger.info("Pausing 5 seconds to allow workers to finish...")
-      after(5) do
-        @busy.each(&:terminate)
-        redis.with_connection do |conn|
-          conn.multi do
-            @busy.each do |busy|
-              conn.lpush("queue:#{busy.queue}", busy.msg)
-            end
-          end
+        after(timeout) do
+          @busy.each { |x| x.terminate if x.alive? }
+          signal(:shutdown)
         end
-        signal(:shutdown)
       end
     end
 
@@ -72,11 +74,11 @@ module Sidekiq
       watchdog('sidekiq processor_done crashed!') do
         @done_callback.call(processor) if @done_callback
         @busy.delete(processor)
-        processor.msg = processor.queue = nil
         if stopped?
           processor.terminate if processor.alive?
+          signal(:shutdown) if @busy.empty?
         else
-          @ready << processor
+          @ready << processor if processor.alive?
         end
         dispatch
       end
@@ -88,18 +90,18 @@ module Sidekiq
       unless stopped?
         @ready << Processor.new_link(current_actor)
         dispatch
+      else
+        signal(:shutdown) if @busy.empty?
       end
     end
 
     private
 
     def find_work(queue)
-      msg = redis.lpop("queue:#{queue}")
+      msg = redis { |x| x.lpop("queue:#{queue}") }
       if msg
         processor = @ready.pop
         @busy << processor
-        processor.msg = msg
-        processor.queue = queue
         processor.process!(MultiJson.decode(msg), queue)
       end
       !!msg
@@ -109,6 +111,10 @@ module Sidekiq
       watchdog("Fatal error in sidekiq, dispatch loop died") do
         return if stopped?
 
+        # This is a safety check to ensure we haven't leaked
+        # processors somehow.
+        raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
+
         # Dispatch loop
         loop do
           break logger.debug('no processors') if @ready.empty?