sidekiq 7.3.9 → 8.1.7

data/lib/sidekiq/rails.rb

@@ -1,84 +1,63 @@
 # frozen_string_literal: true
 
-require "sidekiq/job"
-require "rails"
-
 module Sidekiq
-  module ActiveJob
-    # @api private
-    class Wrapper
-      include Sidekiq::Job
-
-      def perform(job_data)
-        ::ActiveJob::Base.execute(job_data.merge("provider_job_id" => jid))
-      end
-    end
-  end
-
-  class Rails < ::Rails::Engine
-    class Reloader
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
-        @app.reloader.wrap(**params) do
-          yield
+  begin
+    gem "railties", ">= 7.0"
+    require "rails"
+    require "sidekiq/job"
+    require_relative "../active_job/queue_adapters/sidekiq_adapter"
+
+    class Rails < ::Rails::Engine
+      class Reloader
+        def initialize(app = ::Rails.application)
+          @app = app
         end
-      end
 
-      def inspect
-        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
-      end
+        def call
+          params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
+          @app.reloader.wrap(**params) do
+            yield
+          end
+        end
 
-      def to_hash
-        {app: @app.class.name}
-      end
-    end
+        def inspect
+          "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+        end
 
-    # By including the Options module, we allow AJs to directly control sidekiq features
-    # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
-    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
-    # manually retried, don't automatically die, etc.
-    #
-    #   class SomeJob < ActiveJob::Base
-    #     queue_as :default
-    #     sidekiq_options retry: 3, backtrace: 10
-    #     def perform
-    #     end
-    #   end
-    initializer "sidekiq.active_job_integration" do
-      ActiveSupport.on_load(:active_job) do
-        require_relative "../active_job/queue_adapters/sidekiq_adapter"
-        include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
+        def to_hash
+          {app: @app.class.name}
+        end
       end
-    end
 
-    initializer "sidekiq.backtrace_cleaner" do
-      Sidekiq.configure_server do |config|
-        config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
+      initializer "sidekiq.backtrace_cleaner" do
+        Sidekiq.configure_server do |config|
+          config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
+        end
       end
-    end
-
-    # This hook happens after all initializers are run, just before returning
-    # from config/environment.rb back to sidekiq/cli.rb.
-    #
-    # None of this matters on the client-side, only within the Sidekiq process itself.
-    config.after_initialize do
-      Sidekiq.configure_server do |config|
-        config[:reloader] = Sidekiq::Rails::Reloader.new
 
-        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
-        # it will appear in the Sidekiq console with all of the job context.
-        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          if ::Rails.logger.respond_to?(:broadcast_to)
-            ::Rails.logger.broadcast_to(config.logger)
-          else
-            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      config.after_initialize do
+        Sidekiq.configure_server do |config|
+          config[:reloader] = Sidekiq::Rails::Reloader.new
+
+          # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
+          # it will appear in the Sidekiq console with all of the job context.
+          unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+            if ::Rails.logger.respond_to?(:broadcast_to)
+              ::Rails.logger.broadcast_to(config.logger)
+            elsif ::ActiveSupport::Logger.respond_to?(:broadcast)
+              ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+            else
+              ::Rails.logger = ::ActiveSupport::BroadcastLogger.new(::Rails.logger, config.logger)
+            end
           end
         end
       end
     end
+  rescue Gem::LoadError
+    # Rails not available or version requirement not met
   end
 end

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
 require "redis_client"
 require "redis_client/decorator"
 
@@ -42,8 +41,8 @@ module Sidekiq
       # this allows us to use methods like `conn.hmset(...)` instead of having to use
       # redis-client's native `conn.call("hmset", ...)`
       def method_missing(*args, &block)
-        warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
-        @client.call(*args, *block)
+        warn("[sidekiq#5788] Redis has deprecated the `#{args.first}` command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
+        @client.call(*args, &block)
       end
       ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
 
@@ -108,6 +107,10 @@ module Sidekiq
       # on by default.
       opts[:reconnect_attempts] ||= 1
 
+      # Identify ourselves to Redis via CLIENT SETINFO so connections
+      # are distinguishable in CLIENT LIST / CLIENT INFO output.
+      opts[:driver_info] ||= "sidekiq_v#{Sidekiq::VERSION}"
+
       opts
     end
   end

data/lib/sidekiq/redis_connection.rb

@@ -10,6 +10,8 @@ module Sidekiq
       def create(options = {})
         symbolized_options = deep_symbolize_keys(options)
         symbolized_options[:url] ||= determine_redis_provider
+        symbolized_options[:password] = wrap(symbolized_options[:password]) if symbolized_options.key?(:password)
+        symbolized_options[:sentinel_password] = wrap(symbolized_options[:sentinel_password]) if symbolized_options.key?(:sentinel_password)
 
         logger = symbolized_options.delete(:logger)
         logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
@@ -21,7 +23,7 @@ module Sidekiq
 
         size = symbolized_options.delete(:size) || 5
         pool_timeout = symbolized_options.delete(:pool_timeout) || 1
-        pool_name = symbolized_options.delete(:pool_name)
+        symbolized_options.delete(:pool_name)
 
         # Default timeout in redis-client is 1 second, which can be too aggressive
         # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
@@ -31,13 +33,22 @@ module Sidekiq
         symbolized_options[:timeout] ||= 3
 
         redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
-        ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
+        ConnectionPool.new(timeout: pool_timeout, size: size) do
           redis_config.new_client
         end
       end
 
       private
 
+      # Wrap hard-coded passwords in a Proc to avoid logging the value
+      def wrap(pwd)
+        if pwd.is_a?(String)
+          ->(username) { pwd }
+        else
+          pwd
+        end
+      end
+
       def deep_symbolize_keys(object)
         case object
         when Hash
@@ -57,14 +68,14 @@ module Sidekiq
         # Deep clone so we can muck with these options all we want and exclude
         # params from dump-and-load that may contain objects that Marshal is
         # unable to safely dump.
-        keys = options.keys - [:logger, :ssl_params]
+        keys = options.keys - [:logger, :ssl_params, :password, :sentinel_password]
         scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
         end
-        scrubbed_options[:password] = redacted if scrubbed_options[:password]
-        scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
+        scrubbed_options[:password] = redacted if options.key?(:password)
+        scrubbed_options[:sentinel_password] = redacted if options.key?(:sentinel_password)
         scrubbed_options[:sentinels]&.each do |sentinel|
           if sentinel.is_a?(String)
             if (uri = URI(sentinel)) && uri.password

data/lib/sidekiq/ring_buffer.rb

@@ -6,6 +6,7 @@ module Sidekiq
   class RingBuffer
     include Enumerable
     extend Forwardable
+
     def_delegators :@buf, :[], :each, :size
 
     def initialize(size, default = 0)

data/lib/sidekiq/scheduled.rb

@@ -72,6 +72,7 @@ module Sidekiq
       include Sidekiq::Component
 
       INITIAL_WAIT = 10
+      attr_accessor :rnd
 
       def initialize(config)
         @config = config
@@ -80,6 +81,7 @@ module Sidekiq
         @done = false
         @thread = nil
         @count_calls = 0
+        @rnd = Random.new
       end
 
       # Shut down this instance, will pause until the thread is dead.
@@ -115,9 +117,7 @@ module Sidekiq
       private
 
       def wait
-        @sleeper.pop(random_poll_interval)
-      rescue Timeout::Error
-        # expected
+        @sleeper.pop(timeout: random_poll_interval, exception: false)
       rescue => ex
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
@@ -151,11 +151,11 @@ module Sidekiq
 
         if count < 10
           # For small clusters, calculate a random interval that is ±50% the desired average.
-          interval * rand + interval.to_f / 2
+          interval * @rnd.rand + interval.to_f / 2
         else
           # With 10+ processes, we should have enough randomness to get decent polling
           # across the entire timespan
-          interval * rand
+          interval * @rnd.rand * 2
         end
       end
 
@@ -223,8 +223,7 @@ module Sidekiq
         total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
-        @sleeper.pop(total)
-      rescue Timeout::Error
+        @sleeper.pop(timeout: total, exception: false)
       ensure
         # periodically clean out the `processes` set in Redis which can collect
         # references to dead processes over time. The process count affects how

data/lib/sidekiq/test_api.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq"
+
+module Sidekiq
+  class Testing
+    class TestModeAlreadySetError < RuntimeError; end
+    class << self
+      attr_accessor :__global_test_mode
+
+      # Calling without a block sets the global test mode, affecting
+      # all threads. Calling with a block only affects the current Thread.
+      def __set_test_mode(mode)
+        if block_given?
+          # Reentrant testing modes will lead to a rat's nest of code which is
+          # hard to reason about. You can set the testing mode once globally and
+          # you can override that global setting once per-thread.
+          raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
+
+          self.__local_test_mode = mode
+          begin
+            yield
+          ensure
+            self.__local_test_mode = nil
+          end
+        else
+          self.__global_test_mode = mode
+        end
+      end
+
+      def __test_mode
+        __local_test_mode || __global_test_mode
+      end
+
+      def __local_test_mode
+        Thread.current[:__sidekiq_test_mode]
+      end
+
+      def __local_test_mode=(value)
+        Thread.current[:__sidekiq_test_mode] = value
+      end
+
+      def disable!(&block)
+        __set_test_mode(:disable, &block)
+      end
+
+      def fake!(&block)
+        __set_test_mode(:fake, &block)
+      end
+
+      def inline!(&block)
+        __set_test_mode(:inline, &block)
+      end
+
+      def enabled?
+        __test_mode != :disable
+      end
+
+      def disabled?
+        __test_mode == :disable
+      end
+
+      def fake?
+        __test_mode == :fake
+      end
+
+      def inline?
+        __test_mode == :inline
+      end
+
+      def server_middleware
+        @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
+        yield @server_chain if block_given?
+        @server_chain
+      end
+    end
+  end
+
+  class EmptyQueueError < RuntimeError; end
+
+  module TestingClient
+    private def atomic_push(conn, payloads)
+      if Sidekiq::Testing.fake?
+        payloads.each do |job|
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job["enqueued_at"] = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) unless job["at"]
+          Queues.push(job["queue"], job["class"], job)
+        end
+        true
+      elsif Sidekiq::Testing.inline?
+        payloads.each do |job|
+          klass = Object.const_get(job["class"])
+          job["id"] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
+        end
+        true
+      else
+        super
+      end
+    end
+  end
+
+  Sidekiq::Client.prepend TestingClient
+
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # There are 2 data structures involved in tandem. This is due to the
+    # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
+    # to the array. Because the array was derived from a filter of the total
+    # jobs enqueued, it appeared as though the array didn't change.
+    #
+    # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
+    # on the queue, and another with keys of the job type, so the array for
+    # HardJob.jobs is a straight reference to a real array.
+    #
+    # Queue-based hash:
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::HardJob",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Job-based hash:
+    #
+    # {
+    #   "TestTesting::HardJob"=>[
+    #     {
+    #       "class"=>"TestTesting::HardJob",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardJob.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs_by_queue[queue]
+      end
+
+      def push(queue, klass, job)
+        jobs_by_queue[queue] << job
+        jobs_by_class[klass] << job
+      end
+
+      def jobs_by_queue
+        @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def jobs_by_class
+        @jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+      alias_method :jobs_by_worker, :jobs_by_class
+
+      def delete_for(jid, queue, klass)
+        jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
+        jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
+      end
+
+      def clear_for(queue, klass)
+        jobs_by_queue[queue.to_s].clear
+        jobs_by_class[klass].clear
+      end
+
+      def clear_all
+        jobs_by_queue.clear
+        jobs_by_class.clear
+      end
+    end
+  end
+
+  module Job
+    ##
+    # The Sidekiq testing infrastructure overrides perform_async
+    # so that it does not actually touch the network.  Instead it
+    # stores the asynchronous jobs in a per-class array so that
+    # their presence/absence can be asserted by your tests.
+    #
+    # This is similar to ActionMailer's :test delivery_method and its
+    # ActionMailer::Base.deliveries array.
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, HardJob.jobs.size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, HardJob.jobs.size
+    #   assert_equal :something, HardJob.jobs[0]['args'][0]
+    #
+    # You can also clear and drain all job types:
+    #
+    #   Sidekiq::Job.clear_all # or .drain_all
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Job.clear_all
+    #     end
+    #   end
+    #
+    # or for acceptance testing, i.e. with cucumber:
+    #
+    #   AfterStep do
+    #     Sidekiq::Job.drain_all
+    #   end
+    #
+    #   When I sign up as "foo@example.com"
+    #   Then I should receive a welcome email to "foo@example.com"
+    #
+    module ClassMethods
+      # Queue for this worker
+      def queue
+        get_sidekiq_options["queue"]
+      end
+
+      # Jobs queued for this worker
+      def jobs
+        Queues.jobs_by_class[to_s]
+      end
+
+      # Clear all jobs for this worker
+      def clear
+        Queues.clear_for(queue, to_s)
+      end
+
+      # Drain and run all jobs for this worker
+      def drain
+        while jobs.any?
+          next_job = jobs.first
+          Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
+          process_job(next_job)
+        end
+      end
+
+      # Pop out a single job and perform it
+      def perform_one
+        raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
+        next_job = jobs.first
+        Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
+        process_job(next_job)
+      end
+
+      def process_job(job)
+        instance = new
+        instance.jid = job["jid"]
+        instance.bid = job["bid"] if instance.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(instance, job, job["queue"]) do
+          execute_job(instance, job["args"])
+        end
+      end
+
+      def execute_job(worker, args)
+        worker.perform(*args)
+      end
+    end
+
+    class << self
+      def jobs # :nodoc:
+        Queues.jobs_by_queue.values.flatten
+      end
+
+      # Clear all queued jobs
+      def clear_all
+        Queues.clear_all
+      end
+
+      # Drain (execute) all queued jobs
+      def drain_all
+        while jobs.any?
+          job_classes = jobs.map { |job| job["class"] }.uniq
+
+          job_classes.each do |job_class|
+            Object.const_get(job_class).drain
+          end
+        end
+      end
+    end
+  end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
+      end
+    end
+  end
+end
+
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING # rubocop:disable Style/GlobalVars
+  warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
+end

data/lib/sidekiq/testing/inline.rb

@@ -1,30 +1,2 @@
-# frozen_string_literal: true
-
-require "sidekiq/testing"
-
-##
-# The Sidekiq inline infrastructure overrides perform_async so that it
-# actually calls perform instead. This allows jobs to be run inline in a
-# testing environment.
-#
-# This is similar to `Resque.inline = true` functionality.
-#
-# Example:
-#
-#   require 'sidekiq/testing/inline'
-#
-#   $external_variable = 0
-#
-#   class ExternalJob
-#     include Sidekiq::Job
-#
-#     def perform
-#       $external_variable = 1
-#     end
-#   end
-#
-#   assert_equal 0, $external_variable
-#   ExternalJob.perform_async
-#   assert_equal 1, $external_variable
-#
-Sidekiq::Testing.inline!
+Sidekiq.testing!(:inline)
+warn('⛔️ `require "sidekiq/testing/inline"` is deprecated and will be removed in Sidekiq 9.0. See https://sidekiq.org/wiki/Testing#new-api')