sidekiq 5.0.0

data/lib/sidekiq/rails.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+module Sidekiq
+  class Rails < ::Rails::Engine
+    # We need to setup this up before any application configuration which might
+    # change Sidekiq middleware.
+    #
+    # This hook happens after `Rails::Application` is inherited within
+    # config/application.rb and before config is touched, usually within the
+    # class block. Definitely before config/environments/*.rb and
+    # config/initializers/*.rb.
+    config.before_configuration do
+      if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
+        Sidekiq.server_middleware do |chain|
+          require 'sidekiq/middleware/server/active_record'
+          chain.add Sidekiq::Middleware::Server::ActiveRecord
+        end
+      end
+    end
+
+    config.after_initialize do
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      # We have to add the reloader after initialize to see if cache_classes has
+      # been turned on.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      #
+      Sidekiq.configure_server do |_|
+        if ::Rails::VERSION::MAJOR >= 5
+          Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
+          Psych::Visitors::ToRuby.prepend(Sidekiq::Rails::PsychAutoload)
+        end
+      end
+    end
+
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
+
+    module PsychAutoload
+      def resolve_class(klass_name)
+        klass_name && klass_name.constantize
+      rescue NameError
+        super
+      end
+    end
+  end if defined?(::Rails)
+end

data/lib/sidekiq/redis_connection.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+require 'connection_pool'
+require 'redis'
+require 'uri'
+
+module Sidekiq
+  class RedisConnection
+    class << self
+
+      def create(options={})
+        options = options.symbolize_keys
+
+        options[:url] ||= determine_redis_provider
+
+        size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
+
+        verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
+
+        pool_timeout = options[:pool_timeout] || 1
+        log_info(options)
+
+        ConnectionPool.new(:timeout => pool_timeout, :size => size) do
+          build_client(options)
+        end
+      end
+
+      private
+
+      # Sidekiq needs a lot of concurrent Redis connections.
+      #
+      # We need a connection for each Processor.
+      # We need a connection for Pro's real-time change listener
+      # We need a connection to various features to call Redis every few seconds:
+      #   - the process heartbeat.
+      #   - enterprise's leader election
+      #   - enterprise's cron support
+      def verify_sizing(size, concurrency)
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
+      end
+
+      def build_client(options)
+        namespace = options[:namespace]
+
+        client = Redis.new client_opts(options)
+        if namespace
+          begin
+            require 'redis/namespace'
+            Redis::Namespace.new(namespace, :redis => client)
+          rescue LoadError
+            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
+            exit(-127)
+          end
+        else
+          client
+        end
+      end
+
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
+        end
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:driver] ||= 'ruby'.freeze
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
+
+        opts
+      end
+
+      def log_info(options)
+        # Don't log Redis AUTH password
+        redacted = "REDACTED"
+        scrubbed_options = options.dup
+        if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
+          uri.password = redacted
+          scrubbed_options[:url] = uri.to_s
+        end
+        if scrubbed_options[:password]
+          scrubbed_options[:password] = redacted
+        end
+        if Sidekiq.server?
+          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
+        else
+          Sidekiq.logger.debug("#{Sidekiq::NAME} client with redis options #{scrubbed_options}")
+        end
+      end
+
+      def determine_redis_provider
+        ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+      end
+
+    end
+  end
+end

data/lib/sidekiq/scheduled.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/api'
+
+module Sidekiq
+  module Scheduled
+    SETS = %w(retry schedule)
+
+    class Enq
+      def enqueue_jobs(now=Time.now.to_f.to_s, sorted_sets=SETS)
+        # A job's "score" in Redis is the time at which it should be processed.
+        # Just check Redis for the set of jobs with a timestamp before now.
+        Sidekiq.redis do |conn|
+          sorted_sets.each do |sorted_set|
+            # Get the next item in the queue if it's score (time to execute) is <= now.
+            # We need to go through the list one at a time to reduce the risk of something
+            # going wrong between the time jobs are popped from the scheduled queue and when
+            # they are pushed onto a work queue and losing the jobs.
+            while job = conn.zrangebyscore(sorted_set, '-inf'.freeze, now, :limit => [0, 1]).first do
+
+              # Pop item off the queue and add it to the work queue. If the job can't be popped from
+              # the queue, it's because another process already popped it so we can move on to the
+              # next one.
+              if conn.zrem(sorted_set, job)
+                Sidekiq::Client.push(Sidekiq.load_json(job))
+                Sidekiq::Logging.logger.debug { "enqueued #{sorted_set}: #{job}" }
+              end
+            end
+          end
+        end
+      end
+    end
+
+    ##
+    # The Poller checks Redis every N seconds for jobs in the retry or scheduled
+    # set have passed their timestamp and should be enqueued.  If so, it
+    # just pops the job back onto its original queue so the
+    # workers can pick it up like any other job.
+    class Poller
+      include Util
+
+      INITIAL_WAIT = 10
+
+      def initialize
+        @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+        @sleeper = ConnectionPool::TimedStack.new
+        @done = false
+        @thread = nil
+      end
+
+      # Shut down this instance, will pause until the thread is dead.
+      def terminate
+        @done = true
+        if @thread
+          t = @thread
+          @thread = nil
+          @sleeper << 0
+          t.value
+        end
+      end
+
+      def start
+        @thread ||= safe_thread("scheduler") do
+          initial_wait
+
+          while !@done
+            enqueue
+            wait
+          end
+          Sidekiq.logger.info("Scheduler exiting...")
+        end
+      end
+
+      def enqueue
+        begin
+          @enq.enqueue_jobs
+        rescue => ex
+          # Most likely a problem with redis networking.
+          # Punt and try again at the next interval
+          logger.error ex.message
+          ex.backtrace.each do |bt|
+            logger.error(bt)
+          end
+        end
+      end
+
+      private
+
+      def wait
+        @sleeper.pop(random_poll_interval)
+      rescue Timeout::Error
+        # expected
+      rescue => ex
+        # if poll_interval_average hasn't been calculated yet, we can
+        # raise an error trying to reach Redis.
+        logger.error ex.message
+        logger.error ex.backtrace.first
+        sleep 5
+      end
+
+      # Calculates a random interval that is ±50% the desired average.
+      def random_poll_interval
+        poll_interval_average * rand + poll_interval_average.to_f / 2
+      end
+
+      # We do our best to tune the poll interval to the size of the active Sidekiq
+      # cluster.  If you have 30 processes and poll every 15 seconds, that means one
+      # Sidekiq is checking Redis every 0.5 seconds - way too often for most people
+      # and really bad if the retry or scheduled sets are large.
+      #
+      # Instead try to avoid polling more than once every 15 seconds.  If you have
+      # 30 Sidekiq processes, we'll poll every 30 * 15 or 450 seconds.
+      # To keep things statistically random, we'll sleep a random amount between
+      # 225 and 675 seconds for each poll or 450 seconds on average.  Otherwise restarting
+      # all your Sidekiq processes at the same time will lead to them all polling at
+      # the same time: the thundering herd problem.
+      #
+      # We only do this if poll_interval_average is unset (the default).
+      def poll_interval_average
+        Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
+      end
+
+      # Calculates an average poll interval based on the number of known Sidekiq processes.
+      # This minimizes a single point of failure by dispersing check-ins but without taxing
+      # Redis if you run many Sidekiq processes.
+      def scaled_poll_interval
+        pcount = Sidekiq::ProcessSet.new.size
+        pcount = 1 if pcount == 0
+        pcount * Sidekiq.options[:average_scheduled_poll_interval]
+      end
+
+      def initial_wait
+        # Have all processes sleep between 5-15 seconds.  10 seconds
+        # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+        # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
+        total = 0
+        total += INITIAL_WAIT unless Sidekiq.options[:poll_interval_average]
+        total += (5 * rand)
+
+        @sleeper.pop(total)
+      rescue Timeout::Error
+      end
+
+    end
+  end
+end

data/lib/sidekiq/testing.rb

@@ -0,0 +1,324 @@
+# frozen_string_literal: true
+require 'securerandom'
+require 'sidekiq'
+
+module Sidekiq
+
+  class Testing
+    class << self
+      attr_accessor :__test_mode
+
+      def __set_test_mode(mode)
+        if block_given?
+          current_mode = self.__test_mode
+          begin
+            self.__test_mode = mode
+            yield
+          ensure
+            self.__test_mode = current_mode
+          end
+        else
+          self.__test_mode = mode
+        end
+      end
+
+      def disable!(&block)
+        __set_test_mode(:disable, &block)
+      end
+
+      def fake!(&block)
+        __set_test_mode(:fake, &block)
+      end
+
+      def inline!(&block)
+        __set_test_mode(:inline, &block)
+      end
+
+      def enabled?
+        self.__test_mode != :disable
+      end
+
+      def disabled?
+        self.__test_mode == :disable
+      end
+
+      def fake?
+        self.__test_mode == :fake
+      end
+
+      def inline?
+        self.__test_mode == :inline
+      end
+
+      def server_middleware
+        @server_chain ||= Middleware::Chain.new
+        yield @server_chain if block_given?
+        @server_chain
+      end
+    end
+  end
+
+  # Default to fake testing to keep old behavior
+  Sidekiq::Testing.fake!
+
+  class EmptyQueueError < RuntimeError; end
+
+  class Client
+    alias_method :raw_push_real, :raw_push
+
+    def raw_push(payloads)
+      if Sidekiq::Testing.fake?
+        payloads.each do |job|
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
+          Queues.push(job['queue'], job['class'], job)
+        end
+        true
+      elsif Sidekiq::Testing.inline?
+        payloads.each do |job|
+          klass = job['class'].constantize
+          job['id'] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
+        end
+        true
+      else
+        raw_push_real(payloads)
+      end
+    end
+  end
+
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # There are 2 data structures involved in tandem. This is due to the
+    # Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
+    # to the array. Because the array was dervied from a filter of the total
+    # jobs enqueued, it appeared as though the array didn't change.
+    #
+    # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
+    # on the queue, and another with keys of the worker names, so the array for
+    # QueueWorker.jobs is a straight reference to a real array.
+    #
+    # Queue-based hash:
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Worker-based hash:
+    #
+    # {
+    #   "TestTesting::QueueWorker"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all workers' jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs_by_queue[queue]
+      end
+
+      def push(queue, klass, job)
+        jobs_by_queue[queue] << job
+        jobs_by_worker[klass] << job
+      end
+
+      def jobs_by_queue
+        @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def jobs_by_worker
+        @jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def delete_for(jid, queue, klass)
+        jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
+        jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
+      end
+
+      def clear_for(queue, klass)
+        jobs_by_queue[queue].clear
+        jobs_by_worker[klass].clear
+      end
+
+      def clear_all
+        jobs_by_queue.clear
+        jobs_by_worker.clear
+      end
+    end
+  end
+
+  module Worker
+    ##
+    # The Sidekiq testing infrastructure overrides perform_async
+    # so that it does not actually touch the network.  Instead it
+    # stores the asynchronous jobs in a per-class array so that
+    # their presence/absence can be asserted by your tests.
+    #
+    # This is similar to ActionMailer's :test delivery_method and its
+    # ActionMailer::Base.deliveries array.
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, HardWorker.jobs.size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, HardWorker.jobs.size
+    #   assert_equal :something, HardWorker.jobs[0]['args'][0]
+    #
+    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
+    #   MyMailer.delay.send_welcome_email('foo@example.com')
+    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
+    #
+    # You can also clear and drain all workers' jobs:
+    #
+    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
+    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
+    #
+    #   MyMailer.delay.send_welcome_email('foo@example.com')
+    #   MyModel.delay.do_something_hard
+    #
+    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
+    #   assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
+    #
+    #   Sidekiq::Worker.clear_all # or .drain_all
+    #
+    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
+    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Worker.clear_all
+    #     end
+    #   end
+    #
+    # or for acceptance testing, i.e. with cucumber:
+    #
+    #   AfterStep do
+    #     Sidekiq::Worker.drain_all
+    #   end
+    #
+    #   When I sign up as "foo@example.com"
+    #   Then I should receive a welcome email to "foo@example.com"
+    #
+    module ClassMethods
+
+      # Queue for this worker
+      def queue
+        self.sidekiq_options["queue"]
+      end
+
+      # Jobs queued for this worker
+      def jobs
+        Queues.jobs_by_worker[self.to_s]
+      end
+
+      # Clear all jobs for this worker
+      def clear
+        Queues.clear_for(queue, self.to_s)
+      end
+
+      # Drain and run all jobs for this worker
+      def drain
+        while jobs.any?
+          next_job = jobs.first
+          Queues.delete_for(next_job["jid"], next_job["queue"], self.to_s)
+          process_job(next_job)
+        end
+      end
+
+      # Pop out a single job and perform it
+      def perform_one
+        raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
+        next_job = jobs.first
+        Queues.delete_for(next_job["jid"], queue, self.to_s)
+        process_job(next_job)
+      end
+
+      def process_job(job)
+        worker = new
+        worker.jid = job['jid']
+        worker.bid = job['bid'] if worker.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
+          execute_job(worker, job['args'])
+        end
+      end
+
+      def execute_job(worker, args)
+        worker.perform(*args)
+      end
+    end
+
+    class << self
+      def jobs # :nodoc:
+        Queues.jobs_by_queue.values.flatten
+      end
+
+      # Clear all queued jobs across all workers
+      def clear_all
+        Queues.clear_all
+      end
+
+      # Drain all queued jobs across all workers
+      def drain_all
+        while jobs.any?
+          worker_classes = jobs.map { |job| job["class"] }.uniq
+
+          worker_classes.each do |worker_class|
+            worker_class.constantize.drain
+          end
+        end
+      end
+    end
+  end
+end
+
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
+  puts("**************************************************")
+  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
+  puts("**************************************************")
+end