sidekiq 5.2.10 → 6.0.0.pre1

data/lib/sidekiq/job_retry.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
-require 'sidekiq/scheduled'
-require 'sidekiq/api'
+
+require "sidekiq/scheduled"
+require "sidekiq/api"
 
 module Sidekiq
   ##
@@ -81,22 +82,19 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
-      if msg['retry']
+      if msg["retry"]
         attempt_retry(nil, msg, queue, e)
       else
         Sidekiq.death_handlers.each do |handler|
-          begin
-            handler.call(msg, e)
-          rescue => handler_ex
-            handle_exception(handler_ex, { context: "Error calling death handler", job: msg })
-          end
+          handler.call(msg, e)
+        rescue => handler_ex
+          handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
         end
       end
 
       raise Handled
     end
 
-
     # The local retry support means that any errors that occur within
     # this block can be associated with the given worker instance.
     # This is required to support the `sidekiq_retries_exhausted` block.
@@ -116,11 +114,11 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
-      if msg['retry'] == nil
-        msg['retry'] = worker.class.get_sidekiq_options['retry']
+      if msg["retry"].nil?
+        msg["retry"] = worker.class.get_sidekiq_options["retry"]
       end
 
-      raise e unless msg['retry']
+      raise e unless msg["retry"]
       attempt_retry(worker, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
@@ -133,13 +131,9 @@ module Sidekiq
     # instantiate the worker instance.  All access must be guarded and
     # best effort.
     def attempt_retry(worker, msg, queue, exception)
-      max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+      max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
 
-      msg['queue'] = if msg['retry_queue']
-        msg['retry_queue']
-      else
-        queue
-      end
+      msg["queue"] = (msg["retry_queue"] || queue)
 
       m = exception_message(exception)
       if m.respond_to?(:scrub!)
@@ -147,32 +141,32 @@ module Sidekiq
         m.scrub!
       end
 
-      msg['error_message'] = m
-      msg['error_class'] = exception.class.name
-      count = if msg['retry_count']
-        msg['retried_at'] = Time.now.to_f
-        msg['retry_count'] += 1
+      msg["error_message"] = m
+      msg["error_class"] = exception.class.name
+      count = if msg["retry_count"]
+        msg["retried_at"] = Time.now.to_f
+        msg["retry_count"] += 1
       else
-        msg['failed_at'] = Time.now.to_f
-        msg['retry_count'] = 0
+        msg["failed_at"] = Time.now.to_f
+        msg["retry_count"] = 0
       end
 
-      if msg['backtrace'] == true
-        msg['error_backtrace'] = exception.backtrace
-      elsif !msg['backtrace']
+      if msg["backtrace"] == true
+        msg["error_backtrace"] = exception.backtrace
+      elsif !msg["backtrace"]
         # do nothing
-      elsif msg['backtrace'].to_i != 0
-        msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+      elsif msg["backtrace"].to_i != 0
+        msg["error_backtrace"] = exception.backtrace[0...msg["backtrace"].to_i]
       end
 
       if count < max_retry_attempts
         delay = delay_for(worker, count, exception)
         # Logging here can break retries if the logging device raises ENOSPC #3979
-        #logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+        # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
         retry_at = Time.now.to_f + delay
         payload = Sidekiq.dump_json(msg)
         Sidekiq.redis do |conn|
-          conn.zadd('retry', retry_at.to_s, payload)
+          conn.zadd("retry", retry_at.to_s, payload)
         end
       else
         # Goodbye dear message, you (re)tried your best I'm sure.
@@ -182,25 +176,23 @@ module Sidekiq
 
     def retries_exhausted(worker, msg, exception)
       begin
-        block = worker && worker.sidekiq_retries_exhausted_block
-        block.call(msg, exception) if block
+        block = worker&.sidekiq_retries_exhausted_block
+        block&.call(msg, exception)
       rescue => e
-        handle_exception(e, { context: "Error calling retries_exhausted", job: msg })
+        handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
       Sidekiq.death_handlers.each do |handler|
-        begin
-          handler.call(msg, exception)
-        rescue => e
-          handle_exception(e, { context: "Error calling death handler", job: msg })
-        end
+        handler.call(msg, exception)
+      rescue => e
+        handle_exception(e, {context: "Error calling death handler", job: msg})
       end
 
-      send_to_morgue(msg) unless msg['dead'] == false
+      send_to_morgue(msg) unless msg["dead"] == false
     end
 
     def send_to_morgue(msg)
-      logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+      logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
       payload = Sidekiq.dump_json(msg)
       DeadSet.new.kill(payload, notify_failure: false)
     end
@@ -214,7 +206,7 @@ module Sidekiq
     end
 
     def delay_for(worker, count, exception)
-      if worker && worker.sidekiq_retry_in_block
+      if worker&.sidekiq_retry_in_block
         custom_retry_in = retry_in(worker, count, exception).to_i
         return custom_retry_in if custom_retry_in > 0
       end
@@ -223,16 +215,14 @@ module Sidekiq
 
     # delayed_job uses the same basic formula
     def seconds_to_delay(count)
-      (count ** 4) + 15 + (rand(30)*(count+1))
+      (count**4) + 15 + (rand(30) * (count + 1))
     end
 
     def retry_in(worker, count, exception)
-      begin
-        worker.sidekiq_retry_in_block.call(count, exception)
-      rescue Exception => e
-        handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
-        nil
-      end
+      worker.sidekiq_retry_in_block.call(count, exception)
+    rescue Exception => e
+      handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
+      nil
     end
 
     def exception_caused_by_shutdown?(e, checked_causes = [])
@@ -249,14 +239,11 @@ module Sidekiq
     # Extract message from exception.
     # Set a default if the message raises an error
     def exception_message(exception)
-      begin
-        # App code can stuff all sorts of crazy binary data into the error message
-        # that won't convert to JSON.
-        exception.message.to_s[0, 10_000]
-      rescue
-        "!!! ERROR MESSAGE THREW AN ERROR !!!".dup
-      end
+      # App code can stuff all sorts of crazy binary data into the error message
+      # that won't convert to JSON.
+      exception.message.to_s[0, 10_000]
+    rescue
+      +"!!! ERROR MESSAGE THREW AN ERROR !!!"
     end
-
   end
 end

data/lib/sidekiq/launcher.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
-require 'sidekiq/manager'
-require 'sidekiq/fetch'
-require 'sidekiq/scheduled'
+
+require "sidekiq/manager"
+require "sidekiq/fetch"
+require "sidekiq/scheduled"
 
 module Sidekiq
   # The Launcher is a very simple Actor whose job is to
@@ -11,9 +12,17 @@ module Sidekiq
   class Launcher
     include Util
 
-    attr_accessor :manager, :poller, :fetcher
+    STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
 
-    STATS_TTL = 5*365*24*60*60
+    PROCTITLES = [
+      proc { "sidekiq" },
+      proc { Sidekiq::VERSION },
+      proc { |me, data| data["tag"] },
+      proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
+      proc { |me, data| "stopping" if me.stopping? },
+    ]
+
+    attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
       @manager = Sidekiq::Manager.new(options)
@@ -62,10 +71,30 @@ module Sidekiq
 
     private unless $TESTING
 
+    def start_heartbeat
+      loop do
+        heartbeat
+        sleep 5
+      end
+      Sidekiq.logger.info("Heartbeat stopping...")
+    end
+
+    def clear_heartbeat
+      # Remove record from Redis since we are shutting down.
+      # Note we don't stop the heartbeat thread; if the process
+      # doesn't actually exit, it'll reappear in the Web UI.
+      Sidekiq.redis do |conn|
+        conn.pipelined do
+          conn.srem("processes", identity)
+          conn.del("#{identity}:workers")
+        end
+      end
+    rescue
+      # best effort, ignore network errors
+    end
+
     def heartbeat
-      results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, to_data) }
-      results.compact!
-      $0 = results.join(' ')
+      $0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
 
       ❤
     end
@@ -73,6 +102,7 @@ module Sidekiq
     def ❤
       key = identity
       fails = procd = 0
+
       begin
         fails = Processor::FAILURE.reset
         procd = Processor::PROCESSED.reset
@@ -80,6 +110,7 @@ module Sidekiq
 
         workers_key = "#{key}:workers"
         nowdate = Time.now.utc.strftime("%Y-%m-%d")
+
         Sidekiq.redis do |conn|
           conn.multi do
             conn.incrby("stat:processed", procd)
@@ -97,24 +128,27 @@ module Sidekiq
             conn.expire(workers_key, 60)
           end
         end
+
         fails = procd = 0
 
-        _, exists, _, _, msg = Sidekiq.redis do |conn|
-          conn.multi do
-            conn.sadd('processes', key)
-            conn.exists?(key)
-            conn.hmset(key, 'info', to_json, 'busy', curstate.size, 'beat', Time.now.to_f, 'quiet', @done)
+        _, exists, _, _, msg = Sidekiq.redis { |conn|
+          res = conn.multi {
+            conn.sadd("processes", key)
+            conn.exists(key)
+            conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
-          end
-        end
+          }
+
+          res
+        }
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat
-        fire_event(:heartbeat) if !exists
+        fire_event(:heartbeat) unless exists
 
         return unless msg
 
-        ::Process.kill(msg, $$)
+        ::Process.kill(msg, $PID)
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")
@@ -124,25 +158,17 @@ module Sidekiq
       end
     end
 
-    def start_heartbeat
-      while true
-        heartbeat
-        sleep 5
-      end
-      Sidekiq.logger.info("Heartbeat stopping...")
-    end
-
     def to_data
       @data ||= begin
         {
-          'hostname' => hostname,
-          'started_at' => Time.now.to_f,
-          'pid' => $$,
-          'tag' => @options[:tag] || '',
-          'concurrency' => @options[:concurrency],
-          'queues' => @options[:queues].uniq,
-          'labels' => @options[:labels],
-          'identity' => identity,
+          "hostname" => hostname,
+          "started_at" => Time.now.to_f,
+          "pid" => $PID,
+          "tag" => @options[:tag] || "",
+          "concurrency" => @options[:concurrency],
+          "queues" => @options[:queues].uniq,
+          "labels" => @options[:labels],
+          "identity" => identity,
         }
       end
     end
@@ -154,20 +180,5 @@ module Sidekiq
         Sidekiq.dump_json(to_data)
       end
     end
-
-    def clear_heartbeat
-      # Remove record from Redis since we are shutting down.
-      # Note we don't stop the heartbeat thread; if the process
-      # doesn't actually exit, it'll reappear in the Web UI.
-      Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.srem('processes', identity)
-          conn.del("#{identity}:workers")
-        end
-      end
-    rescue
-      # best effort, ignore network errors
-    end
-
   end
 end

data/lib/sidekiq/logger.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "logger"
+require "time"
+
+module Sidekiq
+  class Logger < ::Logger
+    def initialize(*args)
+      super
+
+      self.formatter = Sidekiq.log_formatter
+    end
+
+    def with_context(hash)
+      ctx.merge!(hash)
+      yield
+    ensure
+      hash.keys.each { |key| ctx.delete(key) }
+    end
+
+    def ctx
+      Thread.current[:sidekiq_context] ||= {}
+    end
+
+    module Formatters
+      class Base < ::Logger::Formatter
+        def tid
+          Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+        end
+
+        def ctx
+          Thread.current[:sidekiq_context] ||= {}
+        end
+
+        def format_context
+          " " + ctx.compact.map { |k, v| "#{k}=#{v}" }.join(" ") if ctx.any?
+        end
+      end
+
+      class Pretty < Base
+        def call(severity, time, program_name, message)
+          "#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+        end
+      end
+
+      class WithoutTimestamp < Pretty
+        def call(severity, time, program_name, message)
+          "pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+        end
+      end
+
+      class JSON < Base
+        def call(severity, time, program_name, message)
+          hash = {
+            ts: time.utc.iso8601(3),
+            pid: ::Process.pid,
+            tid: tid,
+            lvl: severity,
+            msg: message,
+          }
+          c = ctx
+          hash["ctx"] = c unless c.empty?
+
+          Sidekiq.dump_json(hash) << "\n"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/manager.rb

@@ -1,12 +1,11 @@
 # frozen_string_literal: true
-require 'sidekiq/util'
-require 'sidekiq/processor'
-require 'sidekiq/fetch'
-require 'thread'
-require 'set'
 
-module Sidekiq
+require "sidekiq/util"
+require "sidekiq/processor"
+require "sidekiq/fetch"
+require "set"
 
+module Sidekiq
   ##
   # The Manager is the central coordination point in Sidekiq, controlling
   # the lifecycle of the Processors.
@@ -27,7 +26,7 @@ module Sidekiq
     attr_reader :workers
     attr_reader :options
 
-    def initialize(options={})
+    def initialize(options = {})
       logger.debug { options.inspect }
       @options = options
       @count = options[:concurrency] || 10
@@ -132,6 +131,5 @@ module Sidekiq
         processor.kill
       end
     end
-
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Sidekiq
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack
@@ -106,7 +107,7 @@ module Sidekiq
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
-        entries.insert(i+1, new_entry)
+        entries.insert(i + 1, new_entry)
       end
 
       def exists?(klass)

data/lib/sidekiq/middleware/i18n.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 #
 # Simple middleware to save the current locale and restore it when the job executes.
 # Use it by requiring it in your initializer:
@@ -9,19 +10,16 @@ module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.
   class Client
-    def call(worker_class, msg, queue, redis_pool)
-      msg['locale'] ||= I18n.locale
+    def call(_worker, msg, _queue, _redis)
+      msg["locale"] ||= I18n.locale
       yield
     end
   end
 
   # Pull the msg locale out and set the current thread to use it.
   class Server
-    def call(worker, msg, queue)
-      I18n.locale = msg['locale'] || I18n.default_locale
-      yield
-    ensure
-      I18n.locale = I18n.default_locale
+    def call(_worker, msg, _queue, &block)
+      I18n.with_locale(msg.fetch("locale", I18n.default_locale), &block)
     end
   end
 end

data/lib/sidekiq/paginator.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
+
 module Sidekiq
   module Paginator
-
-    def page(key, pageidx=1, page_size=25, opts=nil)
+    def page(key, pageidx = 1, page_size = 25, opts = nil)
       current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
       pageidx = current_page - 1
       total_size = 0
@@ -14,30 +14,29 @@ module Sidekiq
         type = conn.type(key)
 
         case type
-        when 'zset'
+        when "zset"
           rev = opts && opts[:reverse]
-          total_size, items = conn.multi do
+          total_size, items = conn.multi {
             conn.zcard(key)
             if rev
-              conn.zrevrange(key, starting, ending, :with_scores => true)
+              conn.zrevrange(key, starting, ending, with_scores: true)
             else
-              conn.zrange(key, starting, ending, :with_scores => true)
+              conn.zrange(key, starting, ending, with_scores: true)
             end
-          end
+          }
           [current_page, total_size, items]
-        when 'list'
-          total_size, items = conn.multi do
+        when "list"
+          total_size, items = conn.multi {
             conn.llen(key)
             conn.lrange(key, starting, ending)
-          end
+          }
           [current_page, total_size, items]
-        when 'none'
+        when "none"
           [1, 0, []]
         else
           raise "can't page a #{type}"
         end
       end
     end
-
   end
 end