sidekiq 6.2.1 → 6.4.0

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -13,18 +13,20 @@ module Sidekiq
         @opts = options
       end
 
-      def method_missing(name, *args)
+      def method_missing(name, *args, **kwargs)
         # Sidekiq has a limitation in that its message must be JSON.
         # JSON can't round trip real Ruby objects so we use YAML to
         # serialize the objects to a String.  The YAML will be converted
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
-        obj = [@target, name, args]
+        obj = [@target, name, args, kwargs]
         marshalled = ::YAML.dump(obj)
         if marshalled.size > SIZE_LIMIT
           ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
         end
-        @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
+        @performable.client_push({"class" => @performable,
+                                  "args" => [marshalled],
+                                  "display_class" => "#{@target}.#{name}"}.merge(@opts))
       end
     end
   end

data/lib/sidekiq/fetch.rb

@@ -40,7 +40,7 @@ module Sidekiq
       # 4825 Sidekiq Pro with all queues paused will return an
       # empty set of queues with a trailing TIMEOUT value.
       if qs.size <= 1
-        sleep(2)
+        sleep(TIMEOUT)
         return nil
       end
 
@@ -79,9 +79,10 @@ module Sidekiq
       if @strictly_ordered_queues
         @queues
       else
-        queues = @queues.shuffle!.uniq
-        queues << TIMEOUT
-        queues
+        permute = @queues.shuffle
+        permute.uniq!
+        permute << TIMEOUT
+        permute
       end
     end
   end

data/lib/sidekiq/job.rb

@@ -0,0 +1,13 @@
+require "sidekiq/worker"
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Job = Worker
+end

data/lib/sidekiq/job_logger.rb

@@ -38,7 +38,7 @@ module Sidekiq
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
-        class: job_hash["wrapped"] || job_hash["class"],
+        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
         jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]

data/lib/sidekiq/job_retry.rb

@@ -34,9 +34,10 @@ module Sidekiq
   # The job will be retried this number of times before giving up. (If simply
   # 'true', Sidekiq retries 25 times)
   #
-  # We'll add a bit more data to the job to support retries:
+  # Relevant options for job retries:
   #
-  #  * 'queue' - the queue to use
+  #  * 'queue' - the queue for the initial job
+  #  * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
   #  * 'retry_count' - number of times we've retried so far.
   #  * 'error_message' - the message from the exception
   #  * 'error_class' - the exception class
@@ -52,11 +53,12 @@ module Sidekiq
   #
   #   Sidekiq.options[:max_retries] = 7
   #
-  # or limit the number of retries for a particular worker with:
+  # or limit the number of retries for a particular worker and send retries to
+  # a low priority queue with:
   #
   #    class MyWorker
   #      include Sidekiq::Worker
-  #      sidekiq_options :retry => 10
+  #      sidekiq_options retry: 10, retry_queue: 'low'
   #    end
   #
   class JobRetry
@@ -214,16 +216,12 @@ module Sidekiq
     end
 
     def delay_for(worker, count, exception)
+      jitter = rand(10) * (count + 1)
       if worker&.sidekiq_retry_in_block
         custom_retry_in = retry_in(worker, count, exception).to_i
-        return custom_retry_in if custom_retry_in > 0
+        return custom_retry_in + jitter if custom_retry_in > 0
       end
-      seconds_to_delay(count)
-    end
-
-    # delayed_job uses the same basic formula
-    def seconds_to_delay(count)
-      (count**4) + 15 + (rand(30) * (count + 1))
+      (count**4) + 15 + jitter
     end
 
     def retry_in(worker, count, exception)

data/lib/sidekiq/job_util.rb

@@ -0,0 +1,65 @@
+require "securerandom"
+require "time"
+
+module Sidekiq
+  module JobUtil
+    # These functions encapsulate various job utilities.
+    # They must be simple and free from side effects.
+
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+
+      if Sidekiq.options[:on_complex_arguments] == :raise
+        msg = <<~EOM
+          Job arguments to #{item["class"]} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
+          To disable this error, remove `Sidekiq.strict_args!` from your initializer.
+        EOM
+        raise(ArgumentError, msg) unless json_safe?(item)
+      elsif Sidekiq.options[:on_complex_arguments] == :warn
+        Sidekiq.logger.warn <<~EOM unless json_safe?(item)
+          Job arguments to #{item["class"]} do not serialize to JSON safely. This will raise an error in
+          Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
+          by calling `Sidekiq.strict_args!` during Sidekiq initialization.
+        EOM
+      end
+    end
+
+    def normalize_item(item)
+      validate(item)
+
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?(:get_sidekiq_options)
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
+
+      item["class"] = item["class"].to_s
+      item["queue"] = item["queue"].to_s
+      item["jid"] ||= SecureRandom.hex(12)
+      item["created_at"] ||= Time.now.to_f
+
+      item
+    end
+
+    def normalized_hash(item_class)
+      if item_class.is_a?(Class)
+        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
+        item_class.get_sidekiq_options
+      else
+        Sidekiq.default_worker_options
+      end
+    end
+
+    private
+
+    def json_safe?(item)
+      JSON.parse(JSON.dump(item["args"])) == item["args"]
+    end
+  end
+end

data/lib/sidekiq/launcher.rb

@@ -69,10 +69,12 @@ module Sidekiq
 
     private unless $TESTING
 
+    BEAT_PAUSE = 5
+
     def start_heartbeat
       loop do
         heartbeat
-        sleep 5
+        sleep BEAT_PAUSE
       end
       Sidekiq.logger.info("Heartbeat stopping...")
     end
@@ -211,6 +213,8 @@ module Sidekiq
           Your Redis network connection is performing extremely poorly.
           Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
           Ensure Redis is running in the same AZ or datacenter as Sidekiq.
+          If these values are close to 100,000, that means your Sidekiq process may be
+          CPU overloaded; see https://github.com/mperham/sidekiq/discussions/5039
         EOM
         RTT_READINGS.reset
       end
@@ -238,26 +242,22 @@ module Sidekiq
     end
 
     def to_data
-      @data ||= begin
-        {
-          "hostname" => hostname,
-          "started_at" => Time.now.to_f,
-          "pid" => ::Process.pid,
-          "tag" => @options[:tag] || "",
-          "concurrency" => @options[:concurrency],
-          "queues" => @options[:queues].uniq,
-          "labels" => @options[:labels],
-          "identity" => identity
-        }
-      end
+      @data ||= {
+        "hostname" => hostname,
+        "started_at" => Time.now.to_f,
+        "pid" => ::Process.pid,
+        "tag" => @options[:tag] || "",
+        "concurrency" => @options[:concurrency],
+        "queues" => @options[:queues].uniq,
+        "labels" => @options[:labels],
+        "identity" => identity
+      }
     end
 
     def to_json
-      @json ||= begin
-        # this data changes infrequently so dump it to a string
-        # now so we don't need to dump it every heartbeat.
-        Sidekiq.dump_json(to_data)
-      end
+      # this data changes infrequently so dump it to a string
+      # now so we don't need to dump it every heartbeat.
+      @json ||= Sidekiq.dump_json(to_data)
     end
   end
 end

data/lib/sidekiq/manager.rb

@@ -55,9 +55,6 @@ module Sidekiq
       fire_event(:quiet, reverse: true)
     end
 
-    # hack for quicker development / testing environment #2774
-    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
-
     def stop(deadline)
       quiet
       fire_event(:shutdown, reverse: true)
@@ -69,12 +66,7 @@ module Sidekiq
       return if @workers.empty?
 
       logger.info { "Pausing to allow workers to finish..." }
-      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      while remaining > PAUSE_TIME
-        return if @workers.empty?
-        sleep PAUSE_TIME
-        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      end
+      wait_for(deadline) { @workers.empty? }
       return if @workers.empty?
 
       hard_shutdown
@@ -130,6 +122,12 @@ module Sidekiq
       cleanup.each do |processor|
         processor.kill
       end
+
+      # when this method returns, we immediately call `exit` which may not give
+      # the remaining threads time to run `ensure` blocks, etc. We pause here up
+      # to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
+      wait_for(deadline) { @workers.empty? }
     end
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -90,12 +90,12 @@ module Sidekiq
       end
 
       def add(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries << Entry.new(klass, *args)
       end
 
       def prepend(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries.insert(0, Entry.new(klass, *args))
       end
 
@@ -132,7 +132,7 @@ module Sidekiq
       def invoke(*args)
         return yield if empty?
 
-        chain = retrieve.dup
+        chain = retrieve
         traverse_chain = proc do
           if chain.empty?
             yield
@@ -144,6 +144,8 @@ module Sidekiq
       end
     end
 
+    private
+
     class Entry
       attr_reader :klass
 

data/lib/sidekiq/middleware/current_attributes.rb

@@ -0,0 +1,57 @@
+require "active_support/current_attributes"
+
+module Sidekiq
+  ##
+  # Automatically save and load any current attributes in the execution context
+  # so context attributes "flow" from Rails actions into any associated jobs.
+  # This can be useful for multi-tenancy, i18n locale, timezone, any implicit
+  # per-request attribute. See +ActiveSupport::CurrentAttributes+.
+  #
+  # @example
+  #
+  #   # in your initializer
+  #   require "sidekiq/middleware/current_attributes"
+  #   Sidekiq::CurrentAttributes.persist(Myapp::Current)
+  #
+  module CurrentAttributes
+    class Save
+      def initialize(cattr)
+        @klass = cattr
+      end
+
+      def call(_, job, _, _)
+        attrs = @klass.attributes
+        if job.has_key?("cattr")
+          job["cattr"].merge!(attrs)
+        else
+          job["cattr"] = attrs
+        end
+        yield
+      end
+    end
+
+    class Load
+      def initialize(cattr)
+        @klass = cattr
+      end
+
+      def call(_, job, _, &block)
+        if job.has_key?("cattr")
+          @klass.set(job["cattr"], &block)
+        else
+          yield
+        end
+      end
+    end
+
+    def self.persist(klass)
+      Sidekiq.configure_client do |config|
+        config.client_middleware.add Save, klass
+      end
+      Sidekiq.configure_server do |config|
+        config.client_middleware.add Save, klass
+        config.server_middleware.add Load, klass
+      end
+    end
+  end
+end

data/lib/sidekiq/rails.rb

@@ -37,6 +37,17 @@ module Sidekiq
       end
     end
 
+    initializer "sidekiq.rails_logger" do
+      Sidekiq.configure_server do |_|
+        # This is the integration code necessary so that if code uses `Rails.logger.info "Hello"`,
+        # it will appear in the Sidekiq console with all of the job context. See #5021 and
+        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
+        unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
+        end
+      end
+    end
+
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
     #

data/lib/sidekiq/redis_connection.rb

@@ -94,12 +94,10 @@ module Sidekiq
       def log_info(options)
         redacted = "REDACTED"
 
-        # deep clone so we can muck with these options all we want
-        #
-        # exclude SSL params from dump-and-load because some information isn't
-        # safely dumpable in current Rubies
-        keys = options.keys
-        keys.delete(:ssl_params)
+        # Deep clone so we can muck with these options all we want and exclude
+        # params from dump-and-load that may contain objects that Marshal is
+        # unable to safely dump.
+        keys = options.keys - [:logger, :ssl_params]
         scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted

data/lib/sidekiq/scheduled.rb

@@ -9,29 +9,56 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
-      def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
+      LUA_ZPOPBYSCORE = <<~LUA
+        local key, now = KEYS[1], ARGV[1]
+        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        if jobs[1] then
+          redis.call("zrem", key, jobs[1])
+          return jobs[1]
+        end
+      LUA
+
+      def initialize
+        @done = false
+        @lua_zpopbyscore_sha = nil
+      end
+
+      def enqueue_jobs(sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get next items in the queue with scores (time to execute) <= now.
-            until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
-              # We need to go through the list one at a time to reduce the risk of something
-              # going wrong between the time jobs are popped from the scheduled queue and when
-              # they are pushed onto a work queue and losing the jobs.
-              jobs.each do |job|
-                # Pop item off the queue and add it to the work queue. If the job can't be popped from
-                # the queue, it's because another process already popped it so we can move on to the
-                # next one.
-                if conn.zrem(sorted_set, job)
-                  Sidekiq::Client.push(Sidekiq.load_json(job))
-                  Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
-                end
-              end
+            # Get next item in the queue with score (time to execute) <= now.
+            # We need to go through the list one at a time to reduce the risk of something
+            # going wrong between the time jobs are popped from the scheduled queue and when
+            # they are pushed onto a work queue and losing the jobs.
+            while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
+              Sidekiq::Client.push(Sidekiq.load_json(job))
+              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
       end
+
+      def terminate
+        @done = true
+      end
+
+      private
+
+      def zpopbyscore(conn, keys: nil, argv: nil)
+        if @lua_zpopbyscore_sha.nil?
+          raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
+          @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
+        end
+
+        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?("NOSCRIPT")
+
+        @lua_zpopbyscore_sha = nil
+        retry
+      end
     end
 
     ##
@@ -49,11 +76,14 @@ module Sidekiq
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
+        @count_calls = 0
       end
 
       # Shut down this instance, will pause until the thread is dead.
       def terminate
         @done = true
+        @enq.terminate if @enq.respond_to?(:terminate)
+
         if @thread
           t = @thread
           @thread = nil
@@ -152,8 +182,13 @@ module Sidekiq
       end
 
       def process_count
-        pcount = Sidekiq::ProcessSet.new.size
+        # The work buried within Sidekiq::ProcessSet#cleanup can be
+        # expensive at scale. Cut it down by 90% with this counter.
+        # NB: This method is only called by the scheduler thread so we
+        # don't need to worry about the thread safety of +=.
+        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
         pcount = 1 if pcount == 0
+        @count_calls += 1
         pcount
       end
 

data/lib/sidekiq/testing.rb

@@ -338,7 +338,5 @@ module Sidekiq
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
-  puts("**************************************************")
-  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
-  puts("**************************************************")
+  warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end

data/lib/sidekiq/util.rb

@@ -39,6 +39,19 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
+    # hack for quicker development / testing environment #2774
+    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
+
+    # Wait for the orblock to be true or the deadline passed.
+    def wait_for(deadline, &condblock)
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while remaining > PAUSE_TIME
+        return if condblock.call
+        sleep PAUSE_TIME
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.2.1"
+  VERSION = "6.4.0"
 end

data/lib/sidekiq/web/action.rb

@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, [Sidekiq.dump_json(payload)]]
+      [200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -50,7 +50,10 @@ module Sidekiq
 
     get "/" do
       @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
-      stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
+      days = (params["days"] || 30).to_i
+      return halt(401) if days < 1 || days > 180
+
+      stats_history = Sidekiq::Stats::History.new(days)
       @processed_history = stats_history.processed
       @failed_history = stats_history.failed
 
@@ -91,8 +94,8 @@ module Sidekiq
 
       @count = (params["count"] || 25).to_i
       @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
-      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
+      (@current_page, @total_size, @jobs) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
+      @jobs = @jobs.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
 
       erb(:queue)
     end
@@ -113,7 +116,7 @@ module Sidekiq
 
     post "/queues/:name/delete" do
       name = route_params[:name]
-      Sidekiq::Job.new(params["key_val"], name).delete
+      Sidekiq::JobRecord.new(params["key_val"], name).delete
 
       redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
     end
@@ -299,7 +302,7 @@ module Sidekiq
       return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
 
       app = @klass
-      resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
+      resp = catch(:halt) do
         self.class.run_befores(app, action)
         action.instance_exec env, &action.block
       ensure
@@ -314,7 +317,7 @@ module Sidekiq
         # rendered content goes here
         headers = {
           "Content-Type" => "text/html",
-          "Cache-Control" => "no-cache",
+          "Cache-Control" => "private, no-store",
           "Content-Language" => action.locale,
           "Content-Security-Policy" => CSP_HEADER
         }