sidekiq 7.3.10 → 8.0.0.beta1

data/lib/sidekiq/client.rb

@@ -67,9 +67,7 @@ module Sidekiq
         c.pipelined do |p|
           p.hsetnx(key, "cancelled", Time.now.to_i)
           p.hget(key, "cancelled")
-          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
-          # TODO When Redis 7.2 is required
-          # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
         end
       end
       result.to_i
@@ -266,22 +264,21 @@ module Sidekiq
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash["at"].to_s
-          # ActiveJob sets this but the job has not been enqueued yet
-          hash.delete("enqueued_at")
-          # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
-          hash = hash.dup
-          hash.delete("at")
+          # ActiveJob sets enqueued_at but the job has not been enqueued yet
+          hash = hash.except("enqueued_at", "at")
           [at, Sidekiq.dump_json(hash)]
         })
       else
-        queue = payloads.first["queue"]
-        now = Time.now.to_f
-        to_push = payloads.map { |entry|
-          entry["enqueued_at"] = now
-          Sidekiq.dump_json(entry)
-        }
-        conn.sadd("queues", [queue])
-        conn.lpush("queue:#{queue}", to_push)
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) # milliseconds since the epoch
+        grouped_queues = payloads.group_by { |job| job["queue"] }
+        conn.sadd("queues", grouped_queues.keys)
+        grouped_queues.each do |queue, grouped_payloads|
+          to_push = grouped_payloads.map { |entry|
+            entry["enqueued_at"] = now
+            Sidekiq.dump_json(entry)
+          }
+          conn.lpush("queue:#{queue}", to_push)
+        end
       end
     end
   end

data/lib/sidekiq/component.rb

@@ -1,6 +1,23 @@
 # frozen_string_literal: true
 
 module Sidekiq
+  # Ruby's default thread priority is 0, which uses 100ms time slices.
+  # This can lead to some surprising thread starvation; if using a lot of
+  # CPU-heavy concurrency, it may take several seconds before a Thread gets
+  # on the CPU.
+  #
+  # Negative priorities lower the timeslice by half, so -1 = 50ms, -2 = 25ms, etc.
+  # With more frequent timeslices, we reduce the risk of unintentional timeouts
+  # and starvation.
+  #
+  # Customize like so:
+  #
+  #   Sidekiq.configure_server do |cfg|
+  #     cfg.thread_priority = 0
+  #   end
+  #
+  DEFAULT_THREAD_PRIORITY = -1
+
   ##
   # Sidekiq::Component assumes a config instance is available at @config
   module Component # :nodoc:
@@ -13,11 +30,11 @@ module Sidekiq
       raise ex
     end
 
-    def safe_thread(name, &block)
+    def safe_thread(name, priority: nil, &block)
       Thread.new do
         Thread.current.name = "sidekiq.#{name}"
         watchdog(name, &block)
-      end
+      end.tap { |t| t.priority = (priority || config.thread_priority || DEFAULT_THREAD_PRIORITY) }
     end
 
     def logger
@@ -86,5 +103,16 @@ module Sidekiq
         end.join(", ")
       }>"
     end
+
+    def default_tag(dir = Dir.pwd)
+      name = File.basename(dir)
+      prevdir = File.dirname(dir) # Capistrano release directory?
+      if name.to_i != 0 && prevdir
+        if File.basename(prevdir) == "releases"
+          return File.basename(File.dirname(prevdir))
+        end
+      end
+      name
+    end
   end
 end

data/lib/sidekiq/config.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "forwardable"
-
-require "set"
 require "sidekiq/redis_connection"
 
 module Sidekiq
@@ -41,12 +39,22 @@ module Sidekiq
     }
 
     ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
-      l = cfg.logger
-      l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
-      l.warn("#{ex.class.name}: #{ex.message}")
-      unless ex.backtrace.nil?
-        backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
-        l.warn(backtrace.join("\n"))
+      Sidekiq::Context.with(ctx) do
+        dev = cfg[:environment] == "development"
+        fancy = dev && $stdout.tty? # 🎩
+        # Weird logic here but we want to show the backtrace in local
+        # development or if verbose logging is enabled.
+        #
+        # `full_message` contains the error class, message and backtrace
+        # `detailed_message` contains the error class and message
+        #
+        # Absolutely terrible API names. Not useful at all to have two
+        # methods with similar but obscure names.
+        if dev || cfg.logger.debug?
+          cfg.logger.info { ex.full_message(highlight: fancy) }
+        else
+          cfg.logger.info { ex.detailed_message(highlight: fancy) }
+        end
       end
     }
 
@@ -60,6 +68,7 @@ module Sidekiq
 
     def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
     attr_reader :capsules
+    attr_accessor :thread_priority
 
     def inspect
       "#<#{self.class.name} @options=#{
@@ -293,13 +302,7 @@ module Sidekiq
         p ["!!!!!", ex]
       end
       @options[:error_handlers].each do |handler|
-        if parameter_size(handler) == 2
-          # TODO Remove in 8.0
-          logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
-          handler.call(ex, {_config: self}.merge(ctx))
-        else
-          handler.call(ex, ctx, self)
-        end
+        handler.call(ex, ctx, self)
       rescue Exception => e
         l = logger
         l.error "!!! ERROR HANDLER THREW AN ERROR !!!"

data/lib/sidekiq/embedded.rb

@@ -34,6 +34,7 @@ module Sidekiq
     private
 
     def housekeeping
+      @config[:tag] ||= default_tag
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
       logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)

data/lib/sidekiq/fetch.rb

@@ -7,7 +7,6 @@ require "sidekiq/capsule"
 module Sidekiq # :nodoc:
   class BasicFetch
     include Sidekiq::Component
-
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2

data/lib/sidekiq/job/iterable.rb

@@ -54,9 +54,7 @@ module Sidekiq
           c.pipelined do |p|
             p.hsetnx(key, "cancelled", Time.now.to_i)
             p.hget(key, "cancelled")
-            # TODO When Redis 7.2 is required
-            # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
-            p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
+            p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
           end
         end
         @_cancelled = result.to_i
@@ -265,7 +263,7 @@ module Sidekiq
         Sidekiq.redis do |conn|
           conn.multi do |pipe|
             pipe.hset(key, state)
-            pipe.expire(key, STATE_TTL)
+            pipe.expire(key, STATE_TTL, "nx")
             pipe.hget(key, "cancelled")
           end
         end

data/lib/sidekiq/job_retry.rb

@@ -149,7 +149,7 @@ module Sidekiq
 
       m = exception_message(exception)
       if m.respond_to?(:scrub!)
-        m.force_encoding("utf-8")
+        m.force_encoding(Encoding::UTF_8)
         m.scrub!
       end
 
@@ -189,7 +189,7 @@ module Sidekiq
 
       # Logging here can break retries if the logging device raises ENOSPC #3979
       # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
-      jitter = rand(10) * (count + 1)
+      jitter = rand(10 * (count + 1))
       retry_at = Time.now.to_f + delay + jitter
       payload = Sidekiq.dump_json(msg)
       redis do |conn|

data/lib/sidekiq/job_util.rb

@@ -58,10 +58,14 @@ module Sidekiq
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
       item["retry_for"] = item["retry_for"].to_i if item["retry_for"]
-      item["created_at"] ||= Time.now.to_f
+      item["created_at"] ||= now_in_millis
       item
     end
 
+    def now_in_millis
+      ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+    end
+
     def normalized_hash(item_class)
       if item_class.is_a?(Class)
         raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)

data/lib/sidekiq/launcher.rb

@@ -81,7 +81,7 @@ module Sidekiq
       ❤
     end
 
-    private unless $TESTING
+    private
 
     BEAT_PAUSE = 10
 

data/lib/sidekiq/logger.rb

@@ -81,13 +81,9 @@ module Sidekiq
           Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
         end
 
-        def ctx
-          Sidekiq::Context.current
-        end
-
-        def format_context
-          if ctx.any?
-            " " + ctx.compact.map { |k, v|
+        def format_context(ctxt = Sidekiq::Context.current)
+          if ctxt.size > 0
+            ctxt.map { |k, v|
               case v
               when Array
                 "#{k}=#{v.join(",")}"
@@ -101,13 +97,13 @@ module Sidekiq
 
       class Pretty < Base
         def call(severity, time, program_name, message)
-          "#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+          "#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid} #{format_context} #{severity}: #{message}\n"
         end
       end
 
       class WithoutTimestamp < Pretty
         def call(severity, time, program_name, message)
-          "pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+          "pid=#{::Process.pid} tid=#{tid} #{format_context} #{severity}: #{message}\n"
         end
       end
 
@@ -120,7 +116,7 @@ module Sidekiq
             lvl: severity,
             msg: message
           }
-          c = ctx
+          c = Sidekiq::Context.current
           hash["ctx"] = c unless c.empty?
 
           Sidekiq.dump_json(hash) << "\n"

data/lib/sidekiq/manager.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "sidekiq/processor"
-require "set"
 
 module Sidekiq
   ##

data/lib/sidekiq/metrics/query.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require "sidekiq"
 require "date"
-require "set"
-
+require "sidekiq"
 require "sidekiq/metrics/shared"
 
 module Sidekiq

data/lib/sidekiq/middleware/current_attributes.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "active_job/arguments"
 require "active_support/current_attributes"
 
 module Sidekiq
@@ -20,6 +21,8 @@ module Sidekiq
   #   Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
   #
   module CurrentAttributes
+    Serializer = ::ActiveJob::Arguments
+
     class Save
       include Sidekiq::ClientMiddleware
 
@@ -33,26 +36,11 @@ module Sidekiq
             attrs = strklass.constantize.attributes
             # Retries can push the job N times, we don't
             # want retries to reset cattr. #5692, #5090
-            if attrs.any?
-              # Older rails has a bug that `CurrentAttributes#attributes` always returns
-              # the same hash instance. We need to dup it to avoid being accidentally mutated.
-              job[key] = if returns_same_object?
-                attrs.dup
-              else
-                attrs
-              end
-            end
+            job[key] = Serializer.serialize(attrs) if attrs.any?
           end
         end
         yield
       end
-
-      private
-
-      def returns_same_object?
-        ActiveSupport::VERSION::MAJOR < 8 ||
-          (ActiveSupport::VERSION::MAJOR == 8 && ActiveSupport::VERSION::MINOR == 0)
-      end
     end
 
     class Load
@@ -68,7 +56,7 @@ module Sidekiq
         @cattrs.each do |(key, strklass)|
           next unless job.has_key?(key)
 
-          klass_attrs[strklass.constantize] = job[key]
+          klass_attrs[strklass.constantize] = Serializer.deserialize(job[key]).to_h
         end
 
         wrap(klass_attrs.to_a, &block)

data/lib/sidekiq/middleware/i18n.rb

@@ -11,7 +11,6 @@ module Sidekiq::Middleware::I18n
   # to be sent to Sidekiq.
   class Client
     include Sidekiq::ClientMiddleware
-
     def call(_jobclass, job, _queue, _redis)
       job["locale"] ||= I18n.locale
       yield
@@ -21,7 +20,6 @@ module Sidekiq::Middleware::I18n
   # Pull the msg locale out and set the current thread to use it.
   class Server
     include Sidekiq::ServerMiddleware
-
     def call(_jobclass, job, _queue, &block)
       I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
     end

data/lib/sidekiq/paginator.rb

@@ -17,7 +17,14 @@ module Sidekiq
       ending = starting + page_size - 1
 
       Sidekiq.redis do |conn|
-        type = conn.type(key)
+        # horrible, think you can make this cleaner?
+        type = TYPE_CACHE[key]
+        if type
+        elsif key.start_with?("queue:")
+          type = TYPE_CACHE[key] = "list"
+        else
+          type = TYPE_CACHE[key] = conn.type(key)
+        end
         rev = opts && opts[:reverse]
 
         case type

data/lib/sidekiq/processor.rb

@@ -3,6 +3,7 @@
 require "sidekiq/fetch"
 require "sidekiq/job_logger"
 require "sidekiq/job_retry"
+require "sidekiq/profiler"
 
 module Sidekiq
   ##
@@ -66,7 +67,7 @@ module Sidekiq
       @thread ||= safe_thread("#{config.name}/processor", &method(:run))
     end
 
-    private unless $TESTING
+    private
 
     def run
       # By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
@@ -112,13 +113,17 @@ module Sidekiq
     def handle_fetch_exception(ex)
       unless @down
         @down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-        logger.error("Error fetching job: #{ex}")
         handle_exception(ex)
       end
       sleep(1)
       nil
     end
 
+    def profile(job, &block)
+      return yield unless job["profile"]
+      Sidekiq::Profiler.new(config).call(job, &block)
+    end
+
     def dispatch(job_hash, queue, jobstr)
       # since middleware can mutate the job hash
       # we need to clone it to report the original
@@ -132,17 +137,19 @@ module Sidekiq
         @retrier.global(jobstr, queue) do
           @job_logger.call(job_hash, queue) do
             stats(jobstr, queue) do
-              # Rails 5 requires a Reloader to wrap code execution.  In order to
-              # constantize the worker and instantiate an instance, we have to call
-              # the Reloader.  It handles code loading, db connection management, etc.
-              # Effectively this block denotes a "unit of work" to Rails.
-              @reloader.call do
-                klass = Object.const_get(job_hash["class"])
-                instance = klass.new
-                instance.jid = job_hash["jid"]
-                instance._context = self
-                @retrier.local(instance, jobstr, queue) do
-                  yield instance
+              profile(job_hash) do
+                # Rails 5 requires a Reloader to wrap code execution.  In order to
+                # constantize the worker and instantiate an instance, we have to call
+                # the Reloader.  It handles code loading, db connection management, etc.
+                # Effectively this block denotes a "unit of work" to Rails.
+                @reloader.call do
+                  klass = Object.const_get(job_hash["class"])
+                  instance = klass.new
+                  instance.jid = job_hash["jid"]
+                  instance._context = self
+                  @retrier.local(instance, jobstr, queue) do
+                    yield instance
+                  end
                 end
               end
             end
@@ -165,7 +172,6 @@ module Sidekiq
       begin
         job_hash = Sidekiq.load_json(jobstr)
       rescue => ex
-        handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
         now = Time.now.to_f
         redis do |conn|
           conn.multi do |xa|
@@ -174,6 +180,7 @@ module Sidekiq
             xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
           end
         end
+        handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
         return uow.acknowledge
       end
 

data/lib/sidekiq/profiler.rb

@@ -0,0 +1,59 @@
+require "fileutils"
+require "sidekiq/component"
+
+module Sidekiq
+  # Allows the user to profile jobs running in production.
+  # See details in the Profiling wiki page.
+  class Profiler
+    EXPIRY = 86400 # 1 day
+    DEFAULT_OPTIONS = {
+      mode: :wall
+    }
+
+    include Sidekiq::Component
+    def initialize(config)
+      @config = config
+    end
+
+    def call(job, &block)
+      return yield unless job["profile"]
+
+      token = job["profile"]
+      type = job["class"]
+      jid = job["jid"]
+      started_at = Time.now
+      options = DEFAULT_OPTIONS.merge((job["profiler_options"] || {}).transform_keys!(&:to_sym))
+
+      rundata = {
+        started_at: started_at.to_i,
+        token: token,
+        type: type,
+        jid: jid,
+        # .gz extension tells Vernier to compress the data
+        filename: "#{token}-#{type}-#{jid}-#{started_at.strftime("%Y%m%d-%H%M%S")}.json.gz"
+      }
+
+      require "vernier"
+      begin
+        a = Time.now
+        rc = Vernier.profile(**options.merge(out: rundata[:filename]), &block)
+        b = Time.now
+
+        # Failed jobs will raise an exception on previous line and skip this
+        # block. Only successful jobs will persist profile data to Redis.
+        key = "#{token}-#{jid}"
+        data = File.read(rundata[:filename])
+        redis do |conn|
+          conn.multi do |m|
+            m.zadd("profiles", Time.now.to_f + EXPIRY, key)
+            m.hset(key, rundata.merge(elapsed: (b - a), data: data, size: data.bytesize))
+            m.expire(key, EXPIRY)
+          end
+        end
+        rc
+      ensure
+        FileUtils.rm_f(rundata[:filename])
+      end
+    end
+  end
+end

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
 require "redis_client"
 require "redis_client/decorator"
 

data/lib/sidekiq/redis_connection.rb

@@ -10,8 +10,6 @@ module Sidekiq
       def create(options = {})
         symbolized_options = deep_symbolize_keys(options)
         symbolized_options[:url] ||= determine_redis_provider
-        symbolized_options[:password] = wrap(symbolized_options[:password]) if symbolized_options.key?(:password)
-        symbolized_options[:sentinel_password] = wrap(symbolized_options[:sentinel_password]) if symbolized_options.key?(:sentinel_password)
 
         logger = symbolized_options.delete(:logger)
         logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
@@ -40,15 +38,6 @@ module Sidekiq
 
       private
 
-      # Wrap hard-coded passwords in a Proc to avoid logging the value
-      def wrap(pwd)
-        if pwd.is_a?(String)
-          ->(username) { pwd }
-        else
-          pwd
-        end
-      end
-
       def deep_symbolize_keys(object)
         case object
         when Hash
@@ -68,14 +57,14 @@ module Sidekiq
         # Deep clone so we can muck with these options all we want and exclude
         # params from dump-and-load that may contain objects that Marshal is
         # unable to safely dump.
-        keys = options.keys - [:logger, :ssl_params, :password, :sentinel_password]
+        keys = options.keys - [:logger, :ssl_params]
         scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
         end
-        scrubbed_options[:password] = redacted if options.key?(:password)
-        scrubbed_options[:sentinel_password] = redacted if options.key?(:sentinel_password)
+        scrubbed_options[:password] = redacted if scrubbed_options[:password]
+        scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
         scrubbed_options[:sentinels]&.each do |sentinel|
           if sentinel.is_a?(String)
             if (uri = URI(sentinel)) && uri.password

data/lib/sidekiq/ring_buffer.rb

@@ -6,7 +6,6 @@ module Sidekiq
   class RingBuffer
     include Enumerable
     extend Forwardable
-
     def_delegators :@buf, :[], :each, :size
 
     def initialize(size, default = 0)

data/lib/sidekiq/testing.rb

@@ -87,7 +87,7 @@ module Sidekiq
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
-          job["enqueued_at"] = Time.now.to_f unless job["at"]
+          job["enqueued_at"] = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) unless job["at"]
           Queues.push(job["queue"], job["class"], job)
         end
         true
@@ -329,6 +329,6 @@ module Sidekiq
   end
 end
 
-if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING # rubocop:disable Style/GlobalVars
   warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end

data/lib/sidekiq/transaction_aware_client.rb

@@ -7,12 +7,6 @@ module Sidekiq
   class TransactionAwareClient
     def initialize(pool: nil, config: nil)
       @redis_client = Client.new(pool: pool, config: config)
-      @transaction_backend =
-        if ActiveRecord.version >= Gem::Version.new("7.2")
-          ActiveRecord.method(:after_all_transactions_commit)
-        else
-          AfterCommitEverywhere.method(:after_commit)
-        end
     end
 
     def batching?
@@ -26,7 +20,7 @@ module Sidekiq
       # pre-allocate the JID so we can return it immediately and
       # save it to the database as part of the transaction.
       item["jid"] ||= SecureRandom.hex(12)
-      @transaction_backend.call { @redis_client.push(item) }
+      AfterCommitEverywhere.after_commit { @redis_client.push(item) }
       item["jid"]
     end
 
@@ -44,12 +38,10 @@ end
 # Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
 module Sidekiq
   def self.transactional_push!
-    if ActiveRecord.version < Gem::Version.new("7.2")
-      begin
-        require "after_commit_everywhere"
-      rescue LoadError
-        raise %q(You need ActiveRecord >= 7.2 or to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
-      end
+    begin
+      require "after_commit_everywhere"
+    rescue LoadError
+      raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
     end
 
     Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient

data/lib/sidekiq/version.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.3.10"
-  MAJOR = 7
+  VERSION = "8.0.0.beta1"
+  MAJOR = 8
 
   def self.gem_version
     Gem::Version.new(VERSION)