sidekiq 3.4.1 → 4.0.0

data/lib/sidekiq/util.rb

@@ -19,6 +19,12 @@ module Sidekiq
       raise ex
     end
 
+    def safe_thread(name, &block)
+      Thread.new do
+        watchdog(name, &block)
+      end
+    end
+
     def logger
       Sidekiq.logger
     end
@@ -49,6 +55,20 @@ module Sidekiq
           handle_exception(ex, { event: event })
         end
       end
+      arr.clear
+    end
+
+    def want_a_hertz_donut?
+      # what's a hertz donut?
+      # punch!  Hurts, don't it?
+      info = Sidekiq.redis {|c| c.info }
+      if info['connected_clients'].to_i > 1000 && info['hz'].to_i >= 10
+        Sidekiq.logger.warn { "Your Redis `hz` setting is too high at #{info['hz']}.  See mperham/sidekiq#2431.  Set it to 3 in #{info[:config_file]}" }
+        true
+      else
+        Sidekiq.logger.debug { "Redis hz: #{info['hz']}.  Client count: #{info['connected_clients']}" }
+        false
+      end
     end
 
   end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.4.1"
+  VERSION = "4.0.0"
 end

data/lib/sidekiq/web.rb

@@ -11,6 +11,9 @@ module Sidekiq
   class Web < Sinatra::Base
     include Sidekiq::Paginator
 
+    enable :sessions
+    use Rack::Protection, :use => :authenticity_token unless ENV['RACK_ENV'] == 'test'
+
     set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
     set :public_folder, proc { "#{root}/assets" }
     set :views, proc { "#{root}/views" }
@@ -98,7 +101,7 @@ module Sidekiq
     end
 
     post '/morgue' do
-      halt 404 unless params['key']
+      redirect request.path unless params['key']
 
       params['key'].each do |key|
         job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
@@ -259,3 +262,16 @@ module Sidekiq
     end
   end
 end
+
+if defined?(::ActionDispatch::Request::Session) &&
+    !::ActionDispatch::Request::Session.respond_to?(:each)
+  # mperham/sidekiq#2460
+  # Rack apps can't reuse the Rails session store without
+  # this monkeypatch
+  class ActionDispatch::Request::Session
+    def each(&block)
+      hash = self.to_hash
+      hash.each(&block)
+    end
+  end
+end

data/lib/sidekiq/web_helpers.rb

@@ -45,7 +45,7 @@ module Sidekiq
     end
 
     def display_custom_head
-      return unless @head_html
+      return unless defined?(@head_html)
       @head_html.map { |block| capture(&block) }.join
     end
 
@@ -160,7 +160,7 @@ module Sidekiq
       options = options.stringify_keys
       params.merge(options).map do |key, value|
         SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
-      end.join("&")
+      end.compact.join("&")
     end
 
     def truncate(text, truncate_after_chars = 2000)
@@ -169,11 +169,26 @@ module Sidekiq
 
     def display_args(args, truncate_after_chars = 2000)
       args.map do |arg|
-        a = arg.inspect
-        h(truncate(a))
+        h(truncate(to_display(arg)))
       end.join(", ")
     end
 
+    def csrf_tag
+      "<input type='hidden' name='authenticity_token' value='#{session[:csrf]}'/>"
+    end
+
+    def to_display(arg)
+      begin
+        arg.inspect
+      rescue
+        begin
+          arg.to_s
+        rescue => ex
+          "Cannot display argument: [#{ex.class.name}] #{ex.message}"
+        end
+      end
+    end
+
     RETRY_JOB_KEYS = Set.new(%w(
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
@@ -230,5 +245,12 @@ module Sidekiq
     def product_version
       "Sidekiq v#{Sidekiq::VERSION}"
     end
+
+    def redis_connection_and_namespace
+      @redis_connection_and_namespace ||= begin
+        namespace_suffix = namespace == nil ? '' : "##{namespace}"
+        "#{redis_connection}#{namespace_suffix}"
+      end
+    end
   end
 end

data/lib/sidekiq/worker.rb

@@ -24,6 +24,8 @@ module Sidekiq
     attr_accessor :jid
 
     def self.included(base)
+      raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
+
       base.extend(ClassMethods)
       base.class_attribute :sidekiq_options_hash
       base.class_attribute :sidekiq_retry_in_block
@@ -36,6 +38,18 @@ module Sidekiq
 
     module ClassMethods
 
+      def delay(*args)
+        raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
+      end
+
+      def delay_for(*args)
+        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
+      end
+
+      def delay_until(*args)
+        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
+      end
+
       def perform_async(*args)
         client_push('class' => self, 'args' => args)
       end

data/lib/sidekiq.rb

@@ -76,9 +76,19 @@ module Sidekiq
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis(&block)
-    raise ArgumentError, "requires a block" unless block
-    redis_pool.with(&block)
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => ex
+        #2550 Failover can cause the server to become a slave, need
+        # to disconnect and reopen the socket to get back to the master.
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise
+      end
+    end
   end
 
   def self.redis_pool
@@ -100,11 +110,25 @@ module Sidekiq
   end
 
   def self.server_middleware
-    @server_chain ||= Processor.default_middleware
+    @server_chain ||= default_server_middleware
     yield @server_chain if block_given?
     @server_chain
   end
 
+  def self.default_server_middleware
+    require 'sidekiq/middleware/server/retry_jobs'
+    require 'sidekiq/middleware/server/logging'
+
+    Middleware::Chain.new do |m|
+      m.add Middleware::Server::Logging
+      m.add Middleware::Server::RetryJobs
+      if defined?(::ActiveRecord::Base)
+        require 'sidekiq/middleware/server/active_record'
+        m.add Sidekiq::Middleware::Server::ActiveRecord
+      end
+    end
+  end
+
   def self.default_worker_options=(hash)
     @default_worker_options = default_worker_options.merge(hash.stringify_keys)
   end
@@ -129,16 +153,6 @@ module Sidekiq
     Sidekiq::Logging.logger = log
   end
 
-  # When set, overrides Sidekiq.options[:average_scheduled_poll_interval] and sets
-  # the average interval that this process will delay before checking for
-  # scheduled jobs or job retries that are ready to run.
-  #
-  # See sidekiq/scheduled.rb for an in-depth explanation of this value
-  def self.poll_interval=(interval)
-    $stderr.puts "DEPRECATION: `config.poll_interval = #{interval}` will be removed in Sidekiq 4. Please update to `config.average_scheduled_poll_interval = #{interval}`."
-    self.options[:poll_interval_average] = interval
-  end
-
   # How frequently Redis should be checked by a random Sidekiq process for
   # scheduled and retriable jobs. Each individual process will take turns by
   # waiting some multiple of this value.
@@ -172,6 +186,15 @@ module Sidekiq
     raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
     options[:lifecycle_events][event] << block
   end
+
+  # We are shutting down Sidekiq but what about workers that
+  # are working on some long job?  This error is
+  # raised in workers that have not finished within the hard
+  # timeout limit.  This is needed to rollback db transactions,
+  # otherwise Ruby's Thread#kill will commit.  See #377.
+  # DO NOT RESCUE THIS ERROR IN YOUR WORKERS
+  class Shutdown < Interrupt; end
+
 end
 
 require 'sidekiq/extensions/class_methods'

data/sidekiq.gemspec

@@ -4,7 +4,8 @@ require File.expand_path('../lib/sidekiq/version', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["Mike Perham"]
   gem.email         = ["mperham@gmail.com"]
-  gem.description   = gem.summary = "Simple, efficient background processing for Ruby"
+  gem.summary       = "Simple, efficient background processing for Ruby"
+  gem.description   = "Simple, efficient background processing for Ruby."
   gem.homepage      = "http://sidekiq.org"
   gem.license       = "LGPL-3.0"
 
@@ -14,14 +15,13 @@ Gem::Specification.new do |gem|
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
-  gem.add_dependency                  'redis', '>= 3.0.6'
-  gem.add_dependency                  'redis-namespace', '>= 1.3.1'
-  gem.add_dependency                  'connection_pool', '>= 2.1.1'
-  gem.add_dependency                  'celluloid', '~> 0.16.0'
-  gem.add_dependency                  'json'
-  gem.add_development_dependency      'sinatra'
-  gem.add_development_dependency      'minitest', '~> 5.3.3'
-  gem.add_development_dependency      'rake'
-  gem.add_development_dependency      'rails', '~> 4.1.1'
-  gem.add_development_dependency      'coveralls'
+  gem.add_dependency                  'redis', '~> 3.2', '>= 3.2.1'
+  gem.add_dependency                  'connection_pool', '~> 2.2', '>= 2.2.0'
+  gem.add_dependency                  'json', '~> 1.0'
+  gem.add_dependency                  'concurrent-ruby', '~> 1.0'
+  gem.add_development_dependency      'redis-namespace', '~> 1.5', '>= 1.5.2'
+  gem.add_development_dependency      'sinatra', '~> 1.4', '>= 1.4.6'
+  gem.add_development_dependency      'minitest', '~> 5.7', '>= 5.7.0'
+  gem.add_development_dependency      'rake', '~> 10.0'
+  gem.add_development_dependency      'rails', '~> 4', '>= 3.2.0'
 end

data/test/helper.rb

@@ -1,18 +1,32 @@
 $TESTING = true
-require 'coveralls'
-Coveralls.wear! do
-  add_filter "/test/"
-  add_filter "/myapp/"
-end
+# disable minitest/parallel threads
+ENV["N"] = "0"
 
-ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
-if ENV.has_key?("SIMPLECOV")
+if ENV["COVERAGE"]
   require 'simplecov'
   SimpleCov.start do
     add_filter "/test/"
     add_filter "/myapp/"
   end
 end
+ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
+
+trap 'USR1' do
+  threads = Thread.list
+
+  puts
+  puts "=" * 80
+  puts "Received USR1 signal; printing all #{threads.count} thread backtraces."
+
+  threads.each do |thr|
+    description = thr == Thread.main ? "Main thread" : thr.inspect
+    puts
+    puts "#{description} backtrace: "
+    puts thr.backtrace.join("\n")
+  end
+
+  puts "=" * 80
+end
 
 begin
   require 'pry-byebug'
@@ -20,10 +34,7 @@ rescue LoadError
 end
 
 require 'minitest/autorun'
-require 'minitest/pride'
 
-require 'celluloid/test'
-Celluloid.boot
 require 'sidekiq'
 require 'sidekiq/util'
 Sidekiq.logger.level = Logger::ERROR
@@ -37,3 +48,27 @@ REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'testy'
 Sidekiq.configure_client do |config|
   config.redis = { :url => REDIS_URL, :namespace => 'testy' }
 end
+
+def capture_logging(lvl=Logger::INFO)
+  old = Sidekiq.logger
+  begin
+    out = StringIO.new
+    logger = Logger.new(out)
+    logger.level = lvl
+    Sidekiq.logger = logger
+    yield
+    out.string
+  ensure
+    Sidekiq.logger = old
+  end
+end
+
+def with_logging(lvl=Logger::DEBUG)
+  old = Sidekiq.logger.level
+  begin
+    Sidekiq.logger.level = lvl
+    yield
+  ensure
+    Sidekiq.logger.level = old
+  end
+end

data/test/test_actors.rb

@@ -0,0 +1,137 @@
+require_relative 'helper'
+require 'sidekiq/cli'
+require 'sidekiq/fetch'
+require 'sidekiq/scheduled'
+require 'sidekiq/processor'
+
+class TestActors < Sidekiq::Test
+  class JoeWorker
+    include Sidekiq::Worker
+    def perform(slp)
+      raise "boom" if slp == "boom"
+      sleep(slp) if slp > 0
+      $count += 1
+    end
+  end
+
+  describe 'threads' do
+    before do
+      Sidekiq.redis {|c| c.flushdb}
+    end
+
+    describe 'scheduler' do
+      it 'can start and stop' do
+        f = Sidekiq::Scheduled::Poller.new
+        f.start
+        f.terminate
+      end
+
+      it 'can schedule' do
+        ss = Sidekiq::ScheduledSet.new
+        q = Sidekiq::Queue.new
+
+        JoeWorker.perform_in(0.01, 0)
+
+        assert_equal 0, q.size
+        assert_equal 1, ss.size
+
+        sleep 0.015
+        s = Sidekiq::Scheduled::Poller.new
+        s.enqueue
+        assert_equal 1, q.size
+        assert_equal 0, ss.size
+        s.terminate
+      end
+    end
+
+    describe 'processor' do
+      before do
+        $count = 0
+      end
+
+      it 'can start and stop' do
+        f = Sidekiq::Processor.new(Mgr.new)
+        f.terminate
+      end
+
+      class Mgr
+        attr_reader :latest_error
+        attr_reader :mutex
+        attr_reader :cond
+        def initialize
+          @mutex = ::Mutex.new
+          @cond = ::ConditionVariable.new
+        end
+        def processor_died(inst, err)
+          @latest_error = err
+          @mutex.synchronize do
+            @cond.signal
+          end
+        end
+        def processor_stopped(inst)
+          @mutex.synchronize do
+            @cond.signal
+          end
+        end
+        def options
+          { :concurrency => 3, :queues => ['default'] }
+        end
+      end
+
+      it 'can process' do
+        mgr = Mgr.new
+
+        p = Sidekiq::Processor.new(mgr)
+        JoeWorker.perform_async(0)
+
+        a = $count
+        p.process_one
+        b = $count
+        assert_equal a + 1, b
+      end
+
+      it 'deals with errors' do
+        mgr = Mgr.new
+
+        p = Sidekiq::Processor.new(mgr)
+        JoeWorker.perform_async("boom")
+        q = Sidekiq::Queue.new
+        assert_equal 1, q.size
+
+        a = $count
+        mgr.mutex.synchronize do
+          p.start
+          mgr.cond.wait(mgr.mutex)
+        end
+        b = $count
+        assert_equal a, b
+
+        sleep 0.001
+        assert_equal false, p.thread.status
+        p.terminate(true)
+        refute_nil mgr.latest_error
+        assert_equal RuntimeError, mgr.latest_error.class
+      end
+
+      it 'gracefully kills' do
+        mgr = Mgr.new
+
+        p = Sidekiq::Processor.new(mgr)
+        JoeWorker.perform_async(1)
+        q = Sidekiq::Queue.new
+        assert_equal 1, q.size
+
+        a = $count
+        p.start
+        sleep(0.02)
+        p.terminate
+        p.kill(true)
+
+        b = $count
+        assert_equal a, b
+        assert_equal false, p.thread.status
+        refute mgr.latest_error, mgr.latest_error.to_s
+      end
+    end
+  end
+end