sidekiq 3.4.1 → 4.0.2

data/lib/sidekiq/testing.rb

@@ -48,6 +48,12 @@ module Sidekiq
       def inline?
         self.__test_mode == :inline
       end
+
+      def server_middleware
+        @server_chain ||= Middleware::Chain.new
+        yield @server_chain if block_given?
+        @server_chain
+      end
     end
   end
 
@@ -62,15 +68,15 @@ module Sidekiq
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+          Queues.push(job['queue'], job['class'], Sidekiq.load_json(Sidekiq.dump_json(job)))
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          job['jid'] ||= SecureRandom.hex(12)
           klass = job['class'].constantize
-          klass.jobs.unshift Sidekiq.load_json(Sidekiq.dump_json(job))
-          klass.perform_one
+          job['id'] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
         end
         true
       else
@@ -79,6 +85,107 @@ module Sidekiq
     end
   end
 
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # There are 2 data structures involved in tandem. This is due to the
+    # Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
+    # to the array. Because the array was dervied from a filter of the total
+    # jobs enqueued, it appeared as though the array didn't change.
+    #
+    # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
+    # on the queue, and another with keys of the worker names, so the array for
+    # QueueWorker.jobs is a straight reference to a real array.
+    #
+    # Queue-based hash:
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Worker-based hash:
+    #
+    # {
+    #   "TestTesting::QueueWorker"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all workers' jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs_by_queue[queue]
+      end
+
+      def push(queue, klass, job)
+        jobs_by_queue[queue] << job
+        jobs_by_worker[klass] << job
+      end
+
+      def jobs_by_queue
+        @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def jobs_by_worker
+        @jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def delete_for(jid, queue, klass)
+        jobs_by_queue[queue].delete_if { |job| job["jid"] == jid }
+        jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
+      end
+
+      def clear_for(queue, klass)
+        jobs_by_queue[queue].clear
+        jobs_by_worker[klass].clear
+      end
+
+      def clear_all
+        jobs_by_queue.clear
+        jobs_by_worker.clear
+      end
+    end
+  end
+
   module Worker
     ##
     # The Sidekiq testing infrastructure overrides perform_async
@@ -137,34 +244,45 @@ module Sidekiq
     #
     module ClassMethods
 
+      # Queue for this worker
+      def queue
+        self.sidekiq_options["queue"]
+      end
+
       # Jobs queued for this worker
       def jobs
-        Worker.jobs[self]
+        Queues.jobs_by_worker[self.to_s]
       end
 
       # Clear all jobs for this worker
       def clear
-        jobs.clear
+        Queues.clear_for(queue, self.to_s)
       end
 
       # Drain and run all jobs for this worker
       def drain
-        while job = jobs.shift do
-          worker = new
-          worker.jid = job['jid']
-          worker.bid = job['bid'] if worker.respond_to?(:bid=)
-          execute_job(worker, job['args'])
+        while jobs.any?
+          next_job = jobs.first
+          Queues.delete_for(next_job["jid"], queue, self.to_s)
+          process_job(next_job)
         end
       end
 
       # Pop out a single job and perform it
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
-        job = jobs.shift
+        next_job = jobs.first
+        Queues.delete_for(next_job["jid"], queue, self.to_s)
+        process_job(next_job)
+      end
+
+      def process_job(job)
         worker = new
         worker.jid = job['jid']
         worker.bid = job['bid'] if worker.respond_to?(:bid=)
-        execute_job(worker, job['args'])
+        Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
+          execute_job(worker, job['args'])
+        end
       end
 
       def execute_job(worker, args)
@@ -174,18 +292,22 @@ module Sidekiq
 
     class << self
       def jobs # :nodoc:
-        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+        Queues.jobs_by_queue.values.flatten
       end
 
       # Clear all queued jobs across all workers
       def clear_all
-        jobs.clear
+        Queues.clear_all
       end
 
       # Drain all queued jobs across all workers
       def drain_all
-        until jobs.values.all?(&:empty?) do
-          jobs.keys.each(&:drain)
+        while jobs.any?
+          worker_classes = jobs.map { |job| job["class"] }.uniq
+
+          worker_classes.each do |worker_class|
+            worker_class.constantize.drain
+          end
         end
       end
     end

data/lib/sidekiq/util.rb

@@ -19,6 +19,12 @@ module Sidekiq
       raise ex
     end
 
+    def safe_thread(name, &block)
+      Thread.new do
+        watchdog(name, &block)
+      end
+    end
+
     def logger
       Sidekiq.logger
     end
@@ -49,6 +55,20 @@ module Sidekiq
           handle_exception(ex, { event: event })
         end
       end
+      arr.clear
+    end
+
+    def want_a_hertz_donut?
+      # what's a hertz donut?
+      # punch!  Hurts, don't it?
+      info = Sidekiq.redis {|c| c.info }
+      if info['connected_clients'].to_i > 1000 && info['hz'].to_i >= 10
+        Sidekiq.logger.warn { "Your Redis `hz` setting is too high at #{info['hz']}.  See mperham/sidekiq#2431.  Set it to 3 in #{info[:config_file]}" }
+        true
+      else
+        Sidekiq.logger.debug { "Redis hz: #{info['hz']}.  Client count: #{info['connected_clients']}" }
+        false
+      end
     end
 
   end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.4.1"
+  VERSION = "4.0.2"
 end

data/lib/sidekiq/web.rb

@@ -11,6 +11,9 @@ module Sidekiq
   class Web < Sinatra::Base
     include Sidekiq::Paginator
 
+    enable :sessions
+    use ::Rack::Protection, :use => :authenticity_token unless ENV['RACK_ENV'] == 'test'
+
     set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
     set :public_folder, proc { "#{root}/assets" }
     set :views, proc { "#{root}/views" }
@@ -98,7 +101,7 @@ module Sidekiq
     end
 
     post '/morgue' do
-      halt 404 unless params['key']
+      redirect request.path unless params['key']
 
       params['key'].each do |key|
         job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
@@ -259,3 +262,16 @@ module Sidekiq
     end
   end
 end
+
+if defined?(::ActionDispatch::Request::Session) &&
+    !::ActionDispatch::Request::Session.respond_to?(:each)
+  # mperham/sidekiq#2460
+  # Rack apps can't reuse the Rails session store without
+  # this monkeypatch
+  class ActionDispatch::Request::Session
+    def each(&block)
+      hash = self.to_hash
+      hash.each(&block)
+    end
+  end
+end

data/lib/sidekiq/web_helpers.rb

@@ -17,8 +17,13 @@ module Sidekiq
       end
     end
 
+    def clear_caches
+      @@strings = nil
+      @@locale_files = nil
+    end
+
     def locale_files
-      @@locale_files = settings.locales.flat_map do |path|
+      @@locale_files ||= settings.locales.flat_map do |path|
         Dir["#{path}/*.yml"]
       end
     end
@@ -45,7 +50,7 @@ module Sidekiq
     end
 
     def display_custom_head
-      return unless @head_html
+      return unless defined?(@head_html)
       @head_html.map { |block| capture(&block) }.join
     end
 
@@ -160,7 +165,7 @@ module Sidekiq
       options = options.stringify_keys
       params.merge(options).map do |key, value|
         SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
-      end.join("&")
+      end.compact.join("&")
     end
 
     def truncate(text, truncate_after_chars = 2000)
@@ -169,15 +174,31 @@ module Sidekiq
 
     def display_args(args, truncate_after_chars = 2000)
       args.map do |arg|
-        a = arg.inspect
-        h(truncate(a))
+        h(truncate(to_display(arg)))
       end.join(", ")
     end
 
+    def csrf_tag
+      "<input type='hidden' name='authenticity_token' value='#{session[:csrf]}'/>"
+    end
+
+    def to_display(arg)
+      begin
+        arg.inspect
+      rescue
+        begin
+          arg.to_s
+        rescue => ex
+          "Cannot display argument: [#{ex.class.name}] #{ex.message}"
+        end
+      end
+    end
+
     RETRY_JOB_KEYS = Set.new(%w(
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
       error_backtrace enqueued_at retry wrapped
+      created_at
     ))
 
     def retry_extra_items(retry_job)
@@ -230,5 +251,12 @@ module Sidekiq
     def product_version
       "Sidekiq v#{Sidekiq::VERSION}"
     end
+
+    def redis_connection_and_namespace
+      @redis_connection_and_namespace ||= begin
+        namespace_suffix = namespace == nil ? '' : "##{namespace}"
+        "#{redis_connection}#{namespace_suffix}"
+      end
+    end
   end
 end

data/lib/sidekiq/worker.rb

@@ -24,6 +24,8 @@ module Sidekiq
     attr_accessor :jid
 
     def self.included(base)
+      raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
+
       base.extend(ClassMethods)
       base.class_attribute :sidekiq_options_hash
       base.class_attribute :sidekiq_retry_in_block
@@ -36,10 +38,24 @@ module Sidekiq
 
     module ClassMethods
 
+      def delay(*args)
+        raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
+      end
+
+      def delay_for(*args)
+        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
+      end
+
+      def delay_until(*args)
+        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
+      end
+
       def perform_async(*args)
         client_push('class' => self, 'args' => args)
       end
 
+      # +interval+ must be a timestamp, numeric or something that acts
+      #   numeric (like an activesupport time interval).
       def perform_in(interval, *args)
         int = interval.to_f
         now = Time.now

data/lib/sidekiq.rb

@@ -76,9 +76,19 @@ module Sidekiq
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis(&block)
-    raise ArgumentError, "requires a block" unless block
-    redis_pool.with(&block)
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => ex
+        #2550 Failover can cause the server to become a slave, need
+        # to disconnect and reopen the socket to get back to the master.
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise
+      end
+    end
   end
 
   def self.redis_pool
@@ -100,11 +110,25 @@ module Sidekiq
   end
 
   def self.server_middleware
-    @server_chain ||= Processor.default_middleware
+    @server_chain ||= default_server_middleware
     yield @server_chain if block_given?
     @server_chain
   end
 
+  def self.default_server_middleware
+    require 'sidekiq/middleware/server/retry_jobs'
+    require 'sidekiq/middleware/server/logging'
+
+    Middleware::Chain.new do |m|
+      m.add Middleware::Server::Logging
+      m.add Middleware::Server::RetryJobs
+      if defined?(::ActiveRecord::Base)
+        require 'sidekiq/middleware/server/active_record'
+        m.add Sidekiq::Middleware::Server::ActiveRecord
+      end
+    end
+  end
+
   def self.default_worker_options=(hash)
     @default_worker_options = default_worker_options.merge(hash.stringify_keys)
   end
@@ -129,16 +153,6 @@ module Sidekiq
     Sidekiq::Logging.logger = log
   end
 
-  # When set, overrides Sidekiq.options[:average_scheduled_poll_interval] and sets
-  # the average interval that this process will delay before checking for
-  # scheduled jobs or job retries that are ready to run.
-  #
-  # See sidekiq/scheduled.rb for an in-depth explanation of this value
-  def self.poll_interval=(interval)
-    $stderr.puts "DEPRECATION: `config.poll_interval = #{interval}` will be removed in Sidekiq 4. Please update to `config.average_scheduled_poll_interval = #{interval}`."
-    self.options[:poll_interval_average] = interval
-  end
-
   # How frequently Redis should be checked by a random Sidekiq process for
   # scheduled and retriable jobs. Each individual process will take turns by
   # waiting some multiple of this value.
@@ -172,6 +186,15 @@ module Sidekiq
     raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
     options[:lifecycle_events][event] << block
   end
+
+  # We are shutting down Sidekiq but what about workers that
+  # are working on some long job?  This error is
+  # raised in workers that have not finished within the hard
+  # timeout limit.  This is needed to rollback db transactions,
+  # otherwise Ruby's Thread#kill will commit.  See #377.
+  # DO NOT RESCUE THIS ERROR IN YOUR WORKERS
+  class Shutdown < Interrupt; end
+
 end
 
 require 'sidekiq/extensions/class_methods'

data/sidekiq.gemspec

@@ -4,7 +4,8 @@ require File.expand_path('../lib/sidekiq/version', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["Mike Perham"]
   gem.email         = ["mperham@gmail.com"]
-  gem.description   = gem.summary = "Simple, efficient background processing for Ruby"
+  gem.summary       = "Simple, efficient background processing for Ruby"
+  gem.description   = "Simple, efficient background processing for Ruby."
   gem.homepage      = "http://sidekiq.org"
   gem.license       = "LGPL-3.0"
 
@@ -14,14 +15,12 @@ Gem::Specification.new do |gem|
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
-  gem.add_dependency                  'redis', '>= 3.0.6'
-  gem.add_dependency                  'redis-namespace', '>= 1.3.1'
-  gem.add_dependency                  'connection_pool', '>= 2.1.1'
-  gem.add_dependency                  'celluloid', '~> 0.16.0'
-  gem.add_dependency                  'json'
-  gem.add_development_dependency      'sinatra'
-  gem.add_development_dependency      'minitest', '~> 5.3.3'
-  gem.add_development_dependency      'rake'
-  gem.add_development_dependency      'rails', '~> 4.1.1'
-  gem.add_development_dependency      'coveralls'
+  gem.add_dependency                  'redis', '~> 3.2', '>= 3.2.1'
+  gem.add_dependency                  'connection_pool', '~> 2.2', '>= 2.2.0'
+  gem.add_dependency                  'concurrent-ruby', '~> 1.0'
+  gem.add_development_dependency      'redis-namespace', '~> 1.5', '>= 1.5.2'
+  gem.add_development_dependency      'sinatra', '~> 1.4', '>= 1.4.6'
+  gem.add_development_dependency      'minitest', '~> 5.7', '>= 5.7.0'
+  gem.add_development_dependency      'rake', '~> 10.0'
+  gem.add_development_dependency      'rails', '~> 4', '>= 3.2.0'
 end

data/test/helper.rb

@@ -1,18 +1,32 @@
 $TESTING = true
-require 'coveralls'
-Coveralls.wear! do
-  add_filter "/test/"
-  add_filter "/myapp/"
-end
+# disable minitest/parallel threads
+ENV["N"] = "0"
 
-ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
-if ENV.has_key?("SIMPLECOV")
+if ENV["COVERAGE"]
   require 'simplecov'
   SimpleCov.start do
     add_filter "/test/"
     add_filter "/myapp/"
   end
 end
+ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
+
+trap 'USR1' do
+  threads = Thread.list
+
+  puts
+  puts "=" * 80
+  puts "Received USR1 signal; printing all #{threads.count} thread backtraces."
+
+  threads.each do |thr|
+    description = thr == Thread.main ? "Main thread" : thr.inspect
+    puts
+    puts "#{description} backtrace: "
+    puts thr.backtrace.join("\n")
+  end
+
+  puts "=" * 80
+end
 
 begin
   require 'pry-byebug'
@@ -20,10 +34,7 @@ rescue LoadError
 end
 
 require 'minitest/autorun'
-require 'minitest/pride'
 
-require 'celluloid/test'
-Celluloid.boot
 require 'sidekiq'
 require 'sidekiq/util'
 Sidekiq.logger.level = Logger::ERROR
@@ -37,3 +48,27 @@ REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'testy'
 Sidekiq.configure_client do |config|
   config.redis = { :url => REDIS_URL, :namespace => 'testy' }
 end
+
+def capture_logging(lvl=Logger::INFO)
+  old = Sidekiq.logger
+  begin
+    out = StringIO.new
+    logger = Logger.new(out)
+    logger.level = lvl
+    Sidekiq.logger = logger
+    yield
+    out.string
+  ensure
+    Sidekiq.logger = old
+  end
+end
+
+def with_logging(lvl=Logger::DEBUG)
+  old = Sidekiq.logger.level
+  begin
+    Sidekiq.logger.level = lvl
+    yield
+  ensure
+    Sidekiq.logger.level = old
+  end
+end