sidekiq 6.4.2 → 6.5.12

data/lib/sidekiq/middleware/chain.rb

@@ -1,82 +1,102 @@
 # frozen_string_literal: true
 
+require "sidekiq/middleware/modules"
+
 module Sidekiq
   # Middleware is code configured to run before/after
-  # a message is processed.  It is patterned after Rack
+  # a job is processed.  It is patterned after Rack
   # middleware. Middleware exists for the client side
   # (pushing jobs onto the queue) as well as the server
   # side (when jobs are actually processed).
   #
+  # Callers will register middleware Classes and Sidekiq will
+  # create new instances of the middleware for every job. This
+  # is important so that instance state is not shared accidentally
+  # between job executions.
+  #
   # To add middleware for the client:
   #
-  # Sidekiq.configure_client do |config|
-  #   config.client_middleware do |chain|
-  #     chain.add MyClientHook
+  #   Sidekiq.configure_client do |config|
+  #     config.client_middleware do |chain|
+  #       chain.add MyClientHook
+  #     end
   #   end
-  # end
   #
   # To modify middleware for the server, just call
   # with another block:
   #
-  # Sidekiq.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.add MyServerHook
-  #     chain.remove ActiveRecord
+  #   Sidekiq.configure_server do |config|
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #       chain.remove ActiveRecord
+  #     end
   #   end
-  # end
   #
   # To insert immediately preceding another entry:
   #
-  # Sidekiq.configure_client do |config|
-  #   config.client_middleware do |chain|
-  #     chain.insert_before ActiveRecord, MyClientHook
+  #   Sidekiq.configure_client do |config|
+  #     config.client_middleware do |chain|
+  #       chain.insert_before ActiveRecord, MyClientHook
+  #     end
   #   end
-  # end
   #
   # To insert immediately after another entry:
   #
-  # Sidekiq.configure_client do |config|
-  #   config.client_middleware do |chain|
-  #     chain.insert_after ActiveRecord, MyClientHook
+  #   Sidekiq.configure_client do |config|
+  #     config.client_middleware do |chain|
+  #       chain.insert_after ActiveRecord, MyClientHook
+  #     end
   #   end
-  # end
   #
   # This is an example of a minimal server middleware:
   #
-  # class MyServerHook
-  #   def call(job_instance, msg, queue)
-  #     puts "Before job"
-  #     yield
-  #     puts "After job"
+  #   class MyServerHook
+  #     include Sidekiq::ServerMiddleware
+  #
+  #     def call(job_instance, msg, queue)
+  #       logger.info "Before job"
+  #       redis {|conn| conn.get("foo") } # do something in Redis
+  #       yield
+  #       logger.info "After job"
+  #     end
   #   end
-  # end
   #
   # This is an example of a minimal client middleware, note
   # the method must return the result or the job will not push
   # to Redis:
   #
-  # class MyClientHook
-  #   def call(job_class, msg, queue, redis_pool)
-  #     puts "Before push"
-  #     result = yield
-  #     puts "After push"
-  #     result
+  #   class MyClientHook
+  #     include Sidekiq::ClientMiddleware
+  #
+  #     def call(job_class, msg, queue, redis_pool)
+  #       logger.info "Before push"
+  #       result = yield
+  #       logger.info "After push"
+  #       result
+  #     end
   #   end
-  # end
   #
   module Middleware
     class Chain
       include Enumerable
 
+      # A unique instance of the middleware chain is created for
+      # each job executed in order to be thread-safe.
+      # @param copy [Sidekiq::Middleware::Chain] New instance of Chain
+      # @returns nil
       def initialize_copy(copy)
         copy.instance_variable_set(:@entries, entries.dup)
+        nil
       end
 
+      # Iterate through each middleware in the chain
       def each(&block)
         entries.each(&block)
       end
 
-      def initialize
+      # @api private
+      def initialize(config = nil) # :nodoc:
+        @config = config
         @entries = nil
         yield self if block_given?
       end
@@ -85,38 +105,55 @@ module Sidekiq
         @entries ||= []
       end
 
+      # Remove all middleware matching the given Class
+      # @param klass [Class]
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
 
+      # Add the given middleware to the end of the chain.
+      # Sidekiq will call `klass.new(*args)` to create a clean
+      # copy of your middleware for every job executed.
+      #
+      #   chain.add(Statsd::Metrics, { collector: "localhost:8125" })
+      #
+      # @param klass [Class] Your middleware class
+      # @param *args [Array<Object>] Set of arguments to pass to every instance of your middleware
       def add(klass, *args)
         remove(klass)
-        entries << Entry.new(klass, *args)
+        entries << Entry.new(@config, klass, *args)
       end
 
+      # Identical to {#add} except the middleware is added to the front of the chain.
       def prepend(klass, *args)
         remove(klass)
-        entries.insert(0, Entry.new(klass, *args))
+        entries.insert(0, Entry.new(@config, klass, *args))
       end
 
+      # Inserts +newklass+ before +oldklass+ in the chain.
+      # Useful if one middleware must run before another middleware.
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
-        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || 0
         entries.insert(i, new_entry)
       end
 
+      # Inserts +newklass+ after +oldklass+ in the chain.
+      # Useful if one middleware must run after another middleware.
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
-        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
         entries.insert(i + 1, new_entry)
       end
 
+      # @return [Boolean] if the given class is already in the chain
       def exists?(klass)
         any? { |entry| entry.klass == klass }
       end
 
+      # @return [Boolean] if the chain contains no middleware
       def empty?
         @entries.nil? || @entries.empty?
       end
@@ -129,6 +166,8 @@ module Sidekiq
         entries.clear
       end
 
+      # Used by Sidekiq to execute the middleware at runtime
+      # @api private
       def invoke(*args)
         return yield if empty?
 
@@ -146,16 +185,21 @@ module Sidekiq
 
     private
 
+    # Represents each link in the middleware chain
+    # @api private
     class Entry
       attr_reader :klass
 
-      def initialize(klass, *args)
+      def initialize(config, klass, *args)
+        @config = config
         @klass = klass
         @args = args
       end
 
       def make_new
-        @klass.new(*@args)
+        x = @klass.new(*@args)
+        x.config = @config if @config && x.respond_to?(:config=)
+        x
       end
     end
   end

data/lib/sidekiq/middleware/current_attributes.rb

@@ -11,33 +11,39 @@ module Sidekiq
   #
   #   # in your initializer
   #   require "sidekiq/middleware/current_attributes"
-  #   Sidekiq::CurrentAttributes.persist(Myapp::Current)
+  #   Sidekiq::CurrentAttributes.persist("Myapp::Current")
   #
   module CurrentAttributes
     class Save
+      include Sidekiq::ClientMiddleware
+
       def initialize(cattr)
-        @klass = cattr
+        @strklass = cattr
       end
 
       def call(_, job, _, _)
-        attrs = @klass.attributes
-        if job.has_key?("cattr")
-          job["cattr"].merge!(attrs)
-        else
-          job["cattr"] = attrs
+        attrs = @strklass.constantize.attributes
+        if attrs.any?
+          if job.has_key?("cattr")
+            job["cattr"].merge!(attrs)
+          else
+            job["cattr"] = attrs
+          end
         end
         yield
       end
     end
 
     class Load
+      include Sidekiq::ServerMiddleware
+
       def initialize(cattr)
-        @klass = cattr
+        @strklass = cattr
       end
 
       def call(_, job, _, &block)
         if job.has_key?("cattr")
-          @klass.set(job["cattr"], &block)
+          @strklass.constantize.set(job["cattr"], &block)
         else
           yield
         end
@@ -46,11 +52,11 @@ module Sidekiq
 
     def self.persist(klass)
       Sidekiq.configure_client do |config|
-        config.client_middleware.add Save, klass
+        config.client_middleware.add Save, klass.to_s
       end
       Sidekiq.configure_server do |config|
-        config.client_middleware.add Save, klass
-        config.server_middleware.add Load, klass
+        config.client_middleware.add Save, klass.to_s
+        config.server_middleware.add Load, klass.to_s
       end
     end
   end

data/lib/sidekiq/middleware/i18n.rb

@@ -10,6 +10,7 @@ module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.
   class Client
+    include Sidekiq::ClientMiddleware
     def call(_jobclass, job, _queue, _redis)
       job["locale"] ||= I18n.locale
       yield
@@ -18,6 +19,7 @@ module Sidekiq::Middleware::I18n
 
   # Pull the msg locale out and set the current thread to use it.
   class Server
+    include Sidekiq::ServerMiddleware
     def call(_jobclass, job, _queue, &block)
       I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
     end

data/lib/sidekiq/middleware/modules.rb

@@ -0,0 +1,21 @@
+module Sidekiq
+  # Server-side middleware must import this Module in order
+  # to get access to server resources during `call`.
+  module ServerMiddleware
+    attr_accessor :config
+    def redis_pool
+      config.redis_pool
+    end
+
+    def logger
+      config.logger
+    end
+
+    def redis(&block)
+      config.redis(&block)
+    end
+  end
+
+  # no difference for now
+  ClientMiddleware = ServerMiddleware
+end

data/lib/sidekiq/monitor.rb

@@ -101,7 +101,7 @@ class Sidekiq::Monitor
       tags = [
         process["tag"],
         process["labels"],
-        (process["quiet"] == "true" ? "quiet" : nil)
+        ((process["quiet"] == "true") ? "quiet" : nil)
       ].flatten.compact
       tags.any? ? "[#{tags.join("] [")}]" : nil
     end

data/lib/sidekiq/paginator.rb

@@ -3,7 +3,7 @@
 module Sidekiq
   module Paginator
     def page(key, pageidx = 1, page_size = 25, opts = nil)
-      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
       pageidx = current_page - 1
       total_size = 0
       items = []
@@ -19,9 +19,9 @@ module Sidekiq
           total_size, items = conn.multi { |transaction|
             transaction.zcard(key)
             if rev
-              transaction.zrevrange(key, starting, ending, with_scores: true)
+              transaction.zrevrange(key, starting, ending, withscores: true)
             else
-              transaction.zrange(key, starting, ending, with_scores: true)
+              transaction.zrange(key, starting, ending, withscores: true)
             end
           }
           [current_page, total_size, items]
@@ -43,5 +43,13 @@ module Sidekiq
         end
       end
     end
+
+    def page_items(items, pageidx = 1, page_size = 25)
+      current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
+      pageidx = current_page - 1
+      starting = pageidx * page_size
+      items = items.to_a
+      [current_page, items.size, items[starting, page_size]]
+    end
   end
 end

data/lib/sidekiq/processor.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/util"
 require "sidekiq/fetch"
 require "sidekiq/job_logger"
 require "sidekiq/job_retry"
@@ -15,29 +14,30 @@ module Sidekiq
   #   b. run the middleware chain
   #   c. call #perform
   #
-  # A Processor can exit due to shutdown (processor_stopped)
-  # or due to an error during job execution (processor_died)
+  # A Processor can exit due to shutdown or due to
+  # an error during job execution.
   #
   # If an error occurs in the job execution, the
   # Processor calls the Manager to create a new one
   # to replace itself and exits.
   #
   class Processor
-    include Util
+    include Sidekiq::Component
 
     attr_reader :thread
     attr_reader :job
 
-    def initialize(mgr, options)
-      @mgr = mgr
+    def initialize(options, &block)
+      @callback = block
       @down = false
       @done = false
       @job = nil
       @thread = nil
+      @config = options
       @strategy = options[:fetch]
       @reloader = options[:reloader] || proc { |&block| block.call }
       @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
-      @retrier = Sidekiq::JobRetry.new
+      @retrier = Sidekiq::JobRetry.new(options)
     end
 
     def terminate(wait = false)
@@ -66,14 +66,14 @@ module Sidekiq
 
     def run
       process_one until @done
-      @mgr.processor_stopped(self)
+      @callback.call(self)
     rescue Sidekiq::Shutdown
-      @mgr.processor_stopped(self)
+      @callback.call(self)
     rescue Exception => ex
-      @mgr.processor_died(self, ex)
+      @callback.call(self, ex)
     end
 
-    def process_one
+    def process_one(&block)
       @job = fetch
       process(@job) if @job
       @job = nil
@@ -152,15 +152,21 @@ module Sidekiq
         job_hash = Sidekiq.load_json(jobstr)
       rescue => ex
         handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
-        # we can't notify because the job isn't a valid hash payload.
-        DeadSet.new.kill(jobstr, notify_failure: false)
+        now = Time.now.to_f
+        config.redis do |conn|
+          conn.multi do |xa|
+            xa.zadd("dead", now.to_s, jobstr)
+            xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
+            xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+          end
+        end
         return uow.acknowledge
       end
 
       ack = false
       begin
         dispatch(job_hash, queue, jobstr) do |inst|
-          Sidekiq.server_middleware.invoke(inst, job_hash, queue) do
+          @config.server_middleware.invoke(inst, job_hash, queue) do
             execute_job(inst, job_hash["args"])
           end
         end
@@ -174,7 +180,7 @@ module Sidekiq
         # signals that we created a retry successfully.  We can acknowlege the job.
         ack = true
         e = h.cause || h
-        handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
+        handle_exception(e, {context: "Job raised exception", job: job_hash})
         raise e
       rescue Exception => ex
         # Unexpected error!  This is very bad and indicates an exception that got past

data/lib/sidekiq/rails.rb

@@ -37,17 +37,6 @@ module Sidekiq
       end
     end
 
-    initializer "sidekiq.rails_logger" do
-      Sidekiq.configure_server do |_|
-        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
-        # it will appear in the Sidekiq console with all of the job context. See #5021 and
-        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
-        unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
-        end
-      end
-    end
-
     config.before_configuration do
       dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
       dep.deprecate_methods(Sidekiq.singleton_class,
@@ -60,8 +49,18 @@ module Sidekiq
     #
     # None of this matters on the client-side, only within the Sidekiq process itself.
     config.after_initialize do
-      Sidekiq.configure_server do |_|
-        Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
+      Sidekiq.configure_server do |config|
+        config[:reloader] = Sidekiq::Rails::Reloader.new
+
+        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
+        # it will appear in the Sidekiq console with all of the job context.
+        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          if ::Rails::VERSION::STRING < "7.1"
+            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+          else
+            ::Rails.logger.broadcast_to(config.logger)
+          end
+        end
       end
     end
   end

data/lib/sidekiq/redis_client_adapter.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "connection_pool"
+require "redis_client"
+require "redis_client/decorator"
+require "uri"
+
+module Sidekiq
+  class RedisClientAdapter
+    BaseError = RedisClient::Error
+    CommandError = RedisClient::CommandError
+
+    module CompatMethods
+      def info
+        @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
+      end
+
+      def evalsha(sha, keys, argv)
+        @client.call("EVALSHA", sha, keys.size, *keys, *argv)
+      end
+
+      def brpoplpush(*args)
+        @client.blocking_call(false, "BRPOPLPUSH", *args)
+      end
+
+      def brpop(*args)
+        @client.blocking_call(false, "BRPOP", *args)
+      end
+
+      def set(*args)
+        @client.call("SET", *args) { |r| r == "OK" }
+      end
+      ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
+
+      def sismember(*args)
+        @client.call("SISMEMBER", *args) { |c| c > 0 }
+      end
+
+      def exists?(key)
+        @client.call("EXISTS", key) { |c| c > 0 }
+      end
+
+      private
+
+      def method_missing(*args, &block)
+        @client.call(*args, *block)
+      end
+      ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
+
+      def respond_to_missing?(name, include_private = false)
+        super # Appease the linter. We can't tell what is a valid command.
+      end
+    end
+
+    CompatClient = RedisClient::Decorator.create(CompatMethods)
+
+    class CompatClient
+      %i[scan sscan zscan hscan].each do |method|
+        alias_method :"#{method}_each", method
+        undef_method method
+      end
+
+      def disconnect!
+        @client.close
+      end
+
+      def connection
+        {id: @client.id}
+      end
+
+      def redis
+        self
+      end
+
+      def _client
+        @client
+      end
+
+      def message
+        yield nil, @queue.pop
+      end
+
+      # NB: this method does not return
+      def subscribe(chan)
+        @queue = ::Queue.new
+
+        pubsub = @client.pubsub
+        pubsub.call("subscribe", chan)
+
+        loop do
+          evt = pubsub.next_event
+          next if evt.nil?
+          next unless evt[0] == "message" && evt[1] == chan
+
+          (_, _, msg) = evt
+          @queue << msg
+          yield self
+        end
+      end
+    end
+
+    def initialize(options)
+      opts = client_opts(options)
+      @config = if opts.key?(:sentinels)
+        RedisClient.sentinel(**opts)
+      else
+        RedisClient.config(**opts)
+      end
+    end
+
+    def new_client
+      CompatClient.new(@config.new_client)
+    end
+
+    private
+
+    def client_opts(options)
+      opts = options.dup
+
+      if opts[:namespace]
+        Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
+          "Either use the redis adapter or remove the namespace.")
+        Kernel.exit(-127)
+      end
+
+      opts.delete(:size)
+      opts.delete(:pool_timeout)
+
+      if opts[:network_timeout]
+        opts[:timeout] = opts[:network_timeout]
+        opts.delete(:network_timeout)
+      end
+
+      if opts[:driver]
+        opts[:driver] = opts[:driver].to_sym
+      end
+
+      opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
+      opts[:role] = opts[:role].to_sym if opts.key?(:role)
+      opts.delete(:url) if opts.key?(:sentinels)
+
+      # Issue #3303, redis-rb will silently retry an operation.
+      # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+      # is performed twice but I believe this is much, much rarer
+      # than the reconnect silently fixing a problem; we keep it
+      # on by default.
+      opts[:reconnect_attempts] ||= 1
+
+      opts
+    end
+  end
+end
+
+Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter