sidekiq 7.1.4 → 8.0.9

data/lib/sidekiq/rails.rb

@@ -1,73 +1,63 @@
 # frozen_string_literal: true
 
-require "sidekiq/job"
-require "rails"
-
 module Sidekiq
-  class Rails < ::Rails::Engine
-    class Reloader
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
-        @app.reloader.wrap(**params) do
-          yield
+  begin
+    gem "railties", ">= 7.0"
+    require "rails"
+    require "sidekiq/job"
+    require_relative "../active_job/queue_adapters/sidekiq_adapter"
+
+    class Rails < ::Rails::Engine
+      class Reloader
+        def initialize(app = ::Rails.application)
+          @app = app
         end
-      end
 
-      def inspect
-        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
-      end
-
-      def to_json(*)
-        Sidekiq.dump_json(inspect)
-      end
-    end
+        def call
+          params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
+          @app.reloader.wrap(**params) do
+            yield
+          end
+        end
 
-    # By including the Options module, we allow AJs to directly control sidekiq features
-    # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
-    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
-    # manually retried, don't automatically die, etc.
-    #
-    #   class SomeJob < ActiveJob::Base
-    #     queue_as :default
-    #     sidekiq_options retry: 3, backtrace: 10
-    #     def perform
-    #     end
-    #   end
-    initializer "sidekiq.active_job_integration" do
-      ActiveSupport.on_load(:active_job) do
-        include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
-      end
-    end
+        def inspect
+          "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+        end
 
-    initializer "sidekiq.rails_logger" do
-      Sidekiq.configure_server do |config|
-        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
-        # it will appear in the Sidekiq console with all of the job context. See #5021 and
-        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
-        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+        def to_hash
+          {app: @app.class.name}
         end
       end
-    end
 
-    initializer "sidekiq.backtrace_cleaner" do
-      Sidekiq.configure_server do |config|
-        config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
+      initializer "sidekiq.backtrace_cleaner" do
+        Sidekiq.configure_server do |config|
+          config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
+        end
       end
-    end
 
-    # This hook happens after all initializers are run, just before returning
-    # from config/environment.rb back to sidekiq/cli.rb.
-    #
-    # None of this matters on the client-side, only within the Sidekiq process itself.
-    config.after_initialize do
-      Sidekiq.configure_server do |config|
-        config[:reloader] = Sidekiq::Rails::Reloader.new
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      config.after_initialize do
+        Sidekiq.configure_server do |config|
+          config[:reloader] = Sidekiq::Rails::Reloader.new
+
+          # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
+          # it will appear in the Sidekiq console with all of the job context.
+          unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+            if ::Rails.logger.respond_to?(:broadcast_to)
+              ::Rails.logger.broadcast_to(config.logger)
+            elsif ::ActiveSupport::Logger.respond_to?(:broadcast)
+              ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+            else
+              ::Rails.logger = ::ActiveSupport::BroadcastLogger.new(::Rails.logger, config.logger)
+            end
+          end
+        end
       end
     end
+  rescue Gem::LoadError
+    # Rails not available or version requirement not met
   end
 end

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
 require "redis_client"
 require "redis_client/decorator"
 
@@ -21,6 +20,22 @@ module Sidekiq
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
 
+      # this is the set of Redis commands used by Sidekiq. Not guaranteed
+      # to be comprehensive, we use this as a performance enhancement to
+      # avoid calling method_missing on most commands
+      USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
+        get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
+        lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
+        publish rpop rpush sadd scard script set sismember smembers
+        srem ttl type unlink zadd zcard zincrby zrange zrem
+        zremrangebyrank zremrangebyscore]
+
+      USED_COMMANDS.each do |name|
+        define_method(name) do |*args, **kwargs|
+          @client.call(name, *args, **kwargs)
+        end
+      end
+
       private
 
       # this allows us to use methods like `conn.hmset(...)` instead of having to use
@@ -48,6 +63,13 @@ module Sidekiq
       opts = client_opts(options)
       @config = if opts.key?(:sentinels)
         RedisClient.sentinel(**opts)
+      elsif opts.key?(:nodes)
+        # Sidekiq does not support Redis clustering but Sidekiq Enterprise's
+        # rate limiters are cluster-safe so we can scale to millions
+        # of rate limiters using a Redis cluster. This requires the
+        # `redis-cluster-client` gem.
+        # Sidekiq::Limiter.redis = { nodes: [...] }
+        RedisClient.cluster(**opts)
       else
         RedisClient.config(**opts)
       end
@@ -63,8 +85,7 @@ module Sidekiq
       opts = options.dup
 
       if opts[:namespace]
-        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
-          "Either use the redis adapter or remove the namespace."
+        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
       end
 
       opts.delete(:size)
@@ -75,13 +96,9 @@ module Sidekiq
         opts.delete(:network_timeout)
       end
 
-      if opts[:driver]
-        opts[:driver] = opts[:driver].to_sym
-      end
-
       opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
       opts[:role] = opts[:role].to_sym if opts.key?(:role)
-      opts.delete(:url) if opts.key?(:sentinels)
+      opts[:driver] = opts[:driver].to_sym if opts.key?(:driver)
 
       # Issue #3303, redis-rb will silently retry an operation.
       # This can lead to duplicate jobs if Sidekiq::Client's LPUSH

data/lib/sidekiq/redis_connection.rb

@@ -8,16 +8,30 @@ module Sidekiq
   module RedisConnection
     class << self
       def create(options = {})
-        symbolized_options = options.transform_keys(&:to_sym)
+        symbolized_options = deep_symbolize_keys(options)
         symbolized_options[:url] ||= determine_redis_provider
+        symbolized_options[:password] = wrap(symbolized_options[:password]) if symbolized_options.key?(:password)
+        symbolized_options[:sentinel_password] = wrap(symbolized_options[:sentinel_password]) if symbolized_options.key?(:sentinel_password)
 
         logger = symbolized_options.delete(:logger)
         logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
 
+        raise "Sidekiq 7+ does not support Redis protocol 2" if symbolized_options[:protocol] == 2
+
+        safe = !!symbolized_options.delete(:cluster_safe)
+        raise ":nodes not allowed, Sidekiq is not safe to run on Redis Cluster" if !safe && symbolized_options.key?(:nodes)
+
         size = symbolized_options.delete(:size) || 5
         pool_timeout = symbolized_options.delete(:pool_timeout) || 1
         pool_name = symbolized_options.delete(:pool_name)
 
+        # Default timeout in redis-client is 1 second, which can be too aggressive
+        # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
+        # it can be easy to get the occasional ReadTimeoutError. You can still provide
+        # a smaller timeout explicitly:
+        #     config.redis = { url: "...", timeout: 1 }
+        symbolized_options[:timeout] ||= 3
+
         redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
         ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
           redis_config.new_client
@@ -26,23 +40,51 @@ module Sidekiq
 
       private
 
+      # Wrap hard-coded passwords in a Proc to avoid logging the value
+      def wrap(pwd)
+        if pwd.is_a?(String)
+          ->(username) { pwd }
+        else
+          pwd
+        end
+      end
+
+      def deep_symbolize_keys(object)
+        case object
+        when Hash
+          object.each_with_object({}) do |(key, value), result|
+            result[key.to_sym] = deep_symbolize_keys(value)
+          end
+        when Array
+          object.map { |e| deep_symbolize_keys(e) }
+        else
+          object
+        end
+      end
+
       def scrub(options)
         redacted = "REDACTED"
 
         # Deep clone so we can muck with these options all we want and exclude
         # params from dump-and-load that may contain objects that Marshal is
         # unable to safely dump.
-        keys = options.keys - [:logger, :ssl_params]
+        keys = options.keys - [:logger, :ssl_params, :password, :sentinel_password]
         scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
         end
-        if scrubbed_options[:password]
-          scrubbed_options[:password] = redacted
-        end
+        scrubbed_options[:password] = redacted if options.key?(:password)
+        scrubbed_options[:sentinel_password] = redacted if options.key?(:sentinel_password)
         scrubbed_options[:sentinels]&.each do |sentinel|
-          sentinel[:password] = redacted if sentinel[:password]
+          if sentinel.is_a?(String)
+            if (uri = URI(sentinel)) && uri.password
+              uri.password = redacted
+              sentinel.replace(uri.to_s)
+            end
+          elsif sentinel[:password]
+            sentinel[:password] = redacted
+          end
         end
         scrubbed_options
       end
@@ -66,9 +108,7 @@ module Sidekiq
           EOM
         end
 
-        ENV[
-          p || "REDIS_URL"
-        ]
+        ENV[p.to_s] || ENV["REDIS_URL"]
       end
     end
   end

data/lib/sidekiq/ring_buffer.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module Sidekiq
   class RingBuffer
     include Enumerable
     extend Forwardable
+
     def_delegators :@buf, :[], :each, :size
 
     def initialize(size, default = 0)

data/lib/sidekiq/scheduled.rb

@@ -144,7 +144,7 @@ module Sidekiq
         # In the example above, each process should schedule every 10 seconds on average. We special
         # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
         # As we run more processes, the scheduling interval average will approach an even spread
-        # between 0 and poll interval so we don't need this artifical boost.
+        # between 0 and poll interval so we don't need this artificial boost.
         #
         count = process_count
         interval = poll_interval_average(count)
@@ -193,7 +193,7 @@ module Sidekiq
       # should never depend on sidekiq/api.
       def cleanup
         # dont run cleanup more than once per minute
-        return 0 unless redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+        return 0 unless redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
 
         count = 0
         redis do |conn|

data/lib/sidekiq/systemd.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Sidekiq's systemd integration allows Sidekiq to inform systemd:
 #  1. when it has successfully started

data/lib/sidekiq/testing.rb

@@ -5,23 +5,42 @@ require "sidekiq"
 
 module Sidekiq
   class Testing
+    class TestModeAlreadySetError < RuntimeError; end
     class << self
-      attr_accessor :__test_mode
+      attr_accessor :__global_test_mode
 
+      # Calling without a block sets the global test mode, affecting
+      # all threads. Calling with a block only affects the current Thread.
       def __set_test_mode(mode)
         if block_given?
-          current_mode = __test_mode
+          # Reentrant testing modes will lead to a rat's nest of code which is
+          # hard to reason about. You can set the testing mode once globally and
+          # you can override that global setting once per-thread.
+          raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
+
+          self.__local_test_mode = mode
           begin
-            self.__test_mode = mode
             yield
           ensure
-            self.__test_mode = current_mode
+            self.__local_test_mode = nil
           end
         else
-          self.__test_mode = mode
+          self.__global_test_mode = mode
         end
       end
 
+      def __test_mode
+        __local_test_mode || __global_test_mode
+      end
+
+      def __local_test_mode
+        Thread.current[:__sidekiq_test_mode]
+      end
+
+      def __local_test_mode=(value)
+        Thread.current[:__sidekiq_test_mode] = value
+      end
+
       def disable!(&block)
         __set_test_mode(:disable, &block)
       end
@@ -64,11 +83,11 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   module TestingClient
-    def raw_push(payloads)
+    private def atomic_push(conn, payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
-          job["enqueued_at"] = Time.now.to_f unless job["at"]
+          job["enqueued_at"] = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) unless job["at"]
           Queues.push(job["queue"], job["class"], job)
         end
         true
@@ -93,7 +112,7 @@ module Sidekiq
     # The Queues class is only for testing the fake queue implementation.
     # There are 2 data structures involved in tandem. This is due to the
     # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
-    # to the array. Because the array was dervied from a filter of the total
+    # to the array. Because the array was derived from a filter of the total
     # jobs enqueued, it appeared as though the array didn't change.
     #
     # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
@@ -259,16 +278,16 @@ module Sidekiq
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
         next_job = jobs.first
-        Queues.delete_for(next_job["jid"], queue, to_s)
+        Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
         process_job(next_job)
       end
 
       def process_job(job)
-        inst = new
-        inst.jid = job["jid"]
-        inst.bid = job["bid"] if inst.respond_to?(:bid=)
-        Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
-          execute_job(inst, job["args"])
+        instance = new
+        instance.jid = job["jid"]
+        instance.bid = job["bid"] if instance.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(instance, job, job["queue"]) do
+          execute_job(instance, job["args"])
         end
       end
 
@@ -310,6 +329,6 @@ module Sidekiq
   end
 end
 
-if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING # rubocop:disable Style/GlobalVars
   warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end

data/lib/sidekiq/transaction_aware_client.rb

@@ -7,13 +7,26 @@ module Sidekiq
   class TransactionAwareClient
     def initialize(pool: nil, config: nil)
       @redis_client = Client.new(pool: pool, config: config)
+      @transaction_backend =
+        if ActiveRecord.version >= Gem::Version.new("7.2")
+          ActiveRecord.method(:after_all_transactions_commit)
+        else
+          AfterCommitEverywhere.method(:after_commit)
+        end
+    end
+
+    def batching?
+      Thread.current[:sidekiq_batch]
     end
 
     def push(item)
+      # 6160 we can't support both Sidekiq::Batch and transactions.
+      return @redis_client.push(item) if batching?
+
       # pre-allocate the JID so we can return it immediately and
       # save it to the database as part of the transaction.
       item["jid"] ||= SecureRandom.hex(12)
-      AfterCommitEverywhere.after_commit { @redis_client.push(item) }
+      @transaction_backend.call { @redis_client.push(item) }
       item["jid"]
     end
 
@@ -31,10 +44,12 @@ end
 # Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
 module Sidekiq
   def self.transactional_push!
-    begin
-      require "after_commit_everywhere"
-    rescue LoadError
-      raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
+    if ActiveRecord.version < Gem::Version.new("7.2")
+      begin
+        require "after_commit_everywhere"
+      rescue LoadError
+        raise %q(You need ActiveRecord >= 7.2 or to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
+      end
     end
 
     Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient

data/lib/sidekiq/version.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.1.4"
-  MAJOR = 7
+  VERSION = "8.0.9"
+  MAJOR = 8
+
+  def self.gem_version
+    Gem::Version.new(VERSION)
+  end
 end