sidekiq 3.5.4 → 5.2.7

data/lib/sidekiq/rails.rb

@@ -1,38 +1,58 @@
-module Sidekiq
-  def self.hook_rails!
-    return if defined?(@delay_removed)
+# frozen_string_literal: true
 
-    ActiveSupport.on_load(:active_record) do
-      include Sidekiq::Extensions::ActiveRecord
+module Sidekiq
+  class Rails < ::Rails::Engine
+    # We need to setup this up before any application configuration which might
+    # change Sidekiq middleware.
+    #
+    # This hook happens after `Rails::Application` is inherited within
+    # config/application.rb and before config is touched, usually within the
+    # class block. Definitely before config/environments/*.rb and
+    # config/initializers/*.rb.
+    config.before_configuration do
+      if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
+        Sidekiq.server_middleware do |chain|
+          require 'sidekiq/middleware/server/active_record'
+          chain.add Sidekiq::Middleware::Server::ActiveRecord
+        end
+      end
     end
 
-    ActiveSupport.on_load(:action_mailer) do
-      extend Sidekiq::Extensions::ActionMailer
+    config.after_initialize do
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      # We have to add the reloader after initialize to see if cache_classes has
+      # been turned on.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      #
+      Sidekiq.configure_server do |_|
+        if ::Rails::VERSION::MAJOR >= 5
+          Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
+        end
+      end
     end
 
-    Module.__send__(:include, Sidekiq::Extensions::Klass)
-  end
-
-  # Removes the generic aliases which MAY clash with names of already
-  #  created methods by other applications. The methods `sidekiq_delay`,
-  #  `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
-  def self.remove_delay!
-    @delay_removed = true
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
 
-    [Extensions::ActiveRecord,
-     Extensions::ActionMailer,
-     Extensions::Klass].each do |mod|
-      mod.module_eval do
-        remove_method :delay if respond_to?(:delay)
-        remove_method :delay_for if respond_to?(:delay_for)
-        remove_method :delay_until if respond_to?(:delay_until)
+      def call
+        @app.reloader.wrap do
+          yield
+        end
       end
-    end
-  end
 
-  class Rails < ::Rails::Engine
-    initializer 'sidekiq' do
-      Sidekiq.hook_rails!
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
     end
   end if defined?(::Rails)
 end
+
+if defined?(::Rails) && ::Rails::VERSION::MAJOR < 4
+  $stderr.puts("**************************************************")
+  $stderr.puts("⛔️ WARNING: Sidekiq server is no longer supported by Rails 3.2 - please ensure your server/workers are updated")
+  $stderr.puts("**************************************************")
+end

data/lib/sidekiq/redis_connection.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'connection_pool'
 require 'redis'
 require 'uri'
@@ -7,12 +8,26 @@ module Sidekiq
     class << self
 
       def create(options={})
+        options.keys.each do |key|
+          options[key.to_sym] = options.delete(key)
+        end
+
+        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{$$}" if !options.has_key?(:id)
         options[:url] ||= determine_redis_provider
 
-        # need a connection for Fetcher and Retry
-        size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 2) : 5)
-        pool_timeout = options[:pool_timeout] || 1
+        size = if options[:size]
+                 options[:size]
+               elsif Sidekiq.server?
+                 Sidekiq.options[:concurrency] + 5
+               elsif ENV['RAILS_MAX_THREADS']
+                 Integer(ENV['RAILS_MAX_THREADS'])
+               else
+                 5
+               end
+
+        verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
+        pool_timeout = options[:pool_timeout] || 1
         log_info(options)
 
         ConnectionPool.new(:timeout => pool_timeout, :size => size) do
@@ -22,13 +37,31 @@ module Sidekiq
 
       private
 
+      # Sidekiq needs a lot of concurrent Redis connections.
+      #
+      # We need a connection for each Processor.
+      # We need a connection for Pro's real-time change listener
+      # We need a connection to various features to call Redis every few seconds:
+      #   - the process heartbeat.
+      #   - enterprise's leader election
+      #   - enterprise's cron support
+      def verify_sizing(size, concurrency)
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size <= concurrency
+      end
+
       def build_client(options)
         namespace = options[:namespace]
 
         client = Redis.new client_opts(options)
         if namespace
-          require 'redis/namespace'
-          Redis::Namespace.new(namespace, :redis => client)
+          begin
+            require 'redis/namespace'
+            Redis::Namespace.new(namespace, :redis => client)
+          rescue LoadError
+            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
+            exit(-127)
+          end
         else
           client
         end
@@ -45,7 +78,14 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] = opts[:driver] || 'ruby'
+        opts[:driver] ||= Redis::Connection.drivers.last || 'ruby'
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
 
         opts
       end
@@ -69,7 +109,34 @@ module Sidekiq
       end
 
       def determine_redis_provider
-        ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+        # If you have this in your environment:
+        # MY_REDIS_URL=redis://hostname.example.com:1238/4
+        # then set:
+        # REDIS_PROVIDER=MY_REDIS_URL
+        # and Sidekiq will find your custom URL variable with no custom
+        # initialization code at all.
+        p = ENV['REDIS_PROVIDER']
+        if p && p =~ /\:/
+          Sidekiq.logger.error <<-EOM
+
+#################################################################################
+
+REDIS_PROVIDER should be set to the **name** of the variable which contains the Redis URL, not a URL itself.
+Platforms like Heroku sell addons that publish a *_URL variable.  You tell Sidekiq with REDIS_PROVIDER, e.g.:
+
+  REDIS_PROVIDER=REDISTOGO_URL
+  REDISTOGO_URL=redis://somehost.example.com:6379/4
+
+Use REDIS_URL if you wish to point Sidekiq to a URL directly.
+
+This configuration error will crash starting in Sidekiq 5.3.
+
+#################################################################################
+EOM
+        end
+        ENV[
+          ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+        ]
       end
 
     end

data/lib/sidekiq/scheduled.rb

@@ -1,6 +1,6 @@
+# frozen_string_literal: true
 require 'sidekiq'
 require 'sidekiq/util'
-require 'sidekiq/actor'
 require 'sidekiq/api'
 
 module Sidekiq
@@ -39,36 +39,92 @@ module Sidekiq
     # workers can pick it up like any other job.
     class Poller
       include Util
-      include Actor
 
       INITIAL_WAIT = 10
 
       def initialize
         @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+        @sleeper = ConnectionPool::TimedStack.new
+        @done = false
+        @thread = nil
       end
 
-      def poll(first_time=false)
-        watchdog('scheduling poller thread died!') do
-          initial_wait if first_time
-
-          begin
-            @enq.enqueue_jobs
-          rescue => ex
-            # Most likely a problem with redis networking.
-            # Punt and try again at the next interval
-            logger.error ex.message
-            logger.error ex.backtrace.first
+      # Shut down this instance, will pause until the thread is dead.
+      def terminate
+        @done = true
+        if @thread
+          t = @thread
+          @thread = nil
+          @sleeper << 0
+          t.value
+        end
+      end
+
+      def start
+        @thread ||= safe_thread("scheduler") do
+          initial_wait
+
+          while !@done
+            enqueue
+            wait
           end
+          Sidekiq.logger.info("Scheduler exiting...")
+        end
+      end
 
-          after(random_poll_interval) { poll }
+      def enqueue
+        begin
+          @enq.enqueue_jobs
+        rescue => ex
+          # Most likely a problem with redis networking.
+          # Punt and try again at the next interval
+          logger.error ex.message
+          handle_exception(ex)
         end
       end
 
       private
 
-      # Calculates a random interval that is ±50% the desired average.
+      def wait
+        @sleeper.pop(random_poll_interval)
+      rescue Timeout::Error
+        # expected
+      rescue => ex
+        # if poll_interval_average hasn't been calculated yet, we can
+        # raise an error trying to reach Redis.
+        logger.error ex.message
+        handle_exception(ex)
+        sleep 5
+      end
+
       def random_poll_interval
-        poll_interval_average * rand + poll_interval_average.to_f / 2
+        # We want one Sidekiq process to schedule jobs every N seconds.  We have M processes
+        # and **don't** want to coordinate.
+        #
+        # So in N*M second timespan, we want each process to schedule once.  The basic loop is:
+        #
+        # * sleep a random amount within that N*M timespan
+        # * wake up and schedule
+        #
+        # We want to avoid one edge case: imagine a set of 2 processes, scheduling every 5 seconds,
+        # so N*M = 10.  Each process decides to randomly sleep 8 seconds, now we've failed to meet
+        # that 5 second average. Thankfully each schedule cycle will sleep randomly so the next
+        # iteration could see each process sleep for 1 second, undercutting our average.
+        #
+        # So below 10 processes, we special case and ensure the processes sleep closer to the average.
+        # In the example above, each process should schedule every 10 seconds on average. We special
+        # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
+        # As we run more processes, the scheduling interval average will approach an even spread
+        # between 0 and poll interval so we don't need this artifical boost.
+        #
+        if process_count < 10
+          # For small clusters, calculate a random interval that is ±50% the desired average.
+          poll_interval_average * rand + poll_interval_average.to_f / 2
+        else
+          # With 10+ processes, we should have enough randomness to get decent polling
+          # across the entire timespan
+          poll_interval_average * rand
+        end
       end
 
       # We do our best to tune the poll interval to the size of the active Sidekiq
@@ -83,7 +139,7 @@ module Sidekiq
       # all your Sidekiq processes at the same time will lead to them all polling at
       # the same time: the thundering herd problem.
       #
-      # We only do this if poll_interval is unset (the default).
+      # We only do this if poll_interval_average is unset (the default).
       def poll_interval_average
         Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
       end
@@ -92,22 +148,25 @@ module Sidekiq
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
       def scaled_poll_interval
+        process_count * Sidekiq.options[:average_scheduled_poll_interval]
+      end
+
+      def process_count
         pcount = Sidekiq::ProcessSet.new.size
         pcount = 1 if pcount == 0
-        pcount * Sidekiq.options[:average_scheduled_poll_interval]
+        pcount
       end
 
       def initial_wait
-        begin
-          # Have all processes sleep between 5-15 seconds.  10 seconds
-          # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
-          # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
-          sleep(INITIAL_WAIT) unless Sidekiq.options[:poll_interval_average]
-          sleep(5 * rand)
-        rescue Celluloid::TaskTerminated
-          # Hit Ctrl-C when Sidekiq is finished booting and we have a chance
-          # to get here.
-        end
+        # Have all processes sleep between 5-15 seconds.  10 seconds
+        # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+        # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
+        total = 0
+        total += INITIAL_WAIT unless Sidekiq.options[:poll_interval_average]
+        total += (5 * rand)
+
+        @sleeper.pop(total)
+      rescue Timeout::Error
       end
 
     end

data/lib/sidekiq/testing.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'securerandom'
 require 'sidekiq'
 
@@ -54,6 +55,15 @@ module Sidekiq
         yield @server_chain if block_given?
         @server_chain
       end
+
+      def constantize(str)
+        names = str.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      end
     end
   end
 
@@ -62,25 +72,128 @@ module Sidekiq
 
   class EmptyQueueError < RuntimeError; end
 
-  class Client
-    alias_method :raw_push_real, :raw_push
-
+  module TestingClient
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
+          Queues.push(job['queue'], job['class'], job)
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          job['jid'] ||= SecureRandom.hex(12)
-          klass = job['class'].constantize
-          klass.jobs.unshift Sidekiq.load_json(Sidekiq.dump_json(job))
-          klass.perform_one
+          klass = Sidekiq::Testing.constantize(job['class'])
+          job['id'] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
         end
         true
       else
-        raw_push_real(payloads)
+        super
+      end
+    end
+  end
+
+  Sidekiq::Client.prepend TestingClient
+
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # There are 2 data structures involved in tandem. This is due to the
+    # Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
+    # to the array. Because the array was dervied from a filter of the total
+    # jobs enqueued, it appeared as though the array didn't change.
+    #
+    # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
+    # on the queue, and another with keys of the worker names, so the array for
+    # QueueWorker.jobs is a straight reference to a real array.
+    #
+    # Queue-based hash:
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Worker-based hash:
+    #
+    # {
+    #   "TestTesting::QueueWorker"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all workers' jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs_by_queue[queue]
+      end
+
+      def push(queue, klass, job)
+        jobs_by_queue[queue] << job
+        jobs_by_worker[klass] << job
+      end
+
+      def jobs_by_queue
+        @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def jobs_by_worker
+        @jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def delete_for(jid, queue, klass)
+        jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
+        jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
+      end
+
+      def clear_for(queue, klass)
+        jobs_by_queue[queue].clear
+        jobs_by_worker[klass].clear
+      end
+
+      def clear_all
+        jobs_by_queue.clear
+        jobs_by_worker.clear
       end
     end
   end
@@ -143,28 +256,36 @@ module Sidekiq
     #
     module ClassMethods
 
+      # Queue for this worker
+      def queue
+        self.sidekiq_options["queue"]
+      end
+
       # Jobs queued for this worker
       def jobs
-        Worker.jobs[self]
+        Queues.jobs_by_worker[self.to_s]
       end
 
       # Clear all jobs for this worker
       def clear
-        jobs.clear
+        Queues.clear_for(queue, self.to_s)
       end
 
       # Drain and run all jobs for this worker
       def drain
-        while job = jobs.shift do
-          process_job(job)
+        while jobs.any?
+          next_job = jobs.first
+          Queues.delete_for(next_job["jid"], next_job["queue"], self.to_s)
+          process_job(next_job)
         end
       end
 
       # Pop out a single job and perform it
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
-        job = jobs.shift
-        process_job(job)
+        next_job = jobs.first
+        Queues.delete_for(next_job["jid"], queue, self.to_s)
+        process_job(next_job)
       end
 
       def process_job(job)
@@ -183,20 +304,30 @@ module Sidekiq
 
     class << self
       def jobs # :nodoc:
-        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+        Queues.jobs_by_queue.values.flatten
       end
 
       # Clear all queued jobs across all workers
       def clear_all
-        jobs.clear
+        Queues.clear_all
       end
 
       # Drain all queued jobs across all workers
       def drain_all
-        until jobs.values.all?(&:empty?) do
-          jobs.keys.each(&:drain)
+        while jobs.any?
+          worker_classes = jobs.map { |job| job["class"] }.uniq
+
+          worker_classes.each do |worker_class|
+            Sidekiq::Testing.constantize(worker_class).drain
+          end
         end
       end
     end
   end
 end
+
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
+  puts("**************************************************")
+  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
+  puts("**************************************************")
+end