sidekiq 6.0.3 → 6.1.0

data/lib/sidekiq/job_logger.rb

@@ -39,7 +39,7 @@ module Sidekiq
       # attribute to expose the underlying thing.
       h = {
         class: job_hash["wrapped"] || job_hash["class"],
-        jid: job_hash["jid"],
+        jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]
       h[:tags] = job_hash["tags"] if job_hash["tags"]

data/lib/sidekiq/job_retry.rb

@@ -189,13 +189,13 @@ module Sidekiq
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
+      send_to_morgue(msg) unless msg["dead"] == false
+
       Sidekiq.death_handlers.each do |handler|
         handler.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling death handler", job: msg})
       end
-
-      send_to_morgue(msg) unless msg["dead"] == false
     end
 
     def send_to_morgue(msg)

data/lib/sidekiq/launcher.rb

@@ -16,12 +16,13 @@ module Sidekiq
       proc { Sidekiq::VERSION },
       proc { |me, data| data["tag"] },
       proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
-      proc { |me, data| "stopping" if me.stopping? },
+      proc { |me, data| "stopping" if me.stopping? }
     ]
 
     attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
+      options[:fetch] ||= BasicFetch.new(options)
       @manager = Sidekiq::Manager.new(options)
       @poller = Sidekiq::Scheduled::Poller.new
       @done = false
@@ -56,7 +57,7 @@ module Sidekiq
 
       # Requeue everything in case there was a worker who grabbed work while stopped
       # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
-      strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+      strategy = @options[:fetch]
       strategy.bulk_requeue([], @options)
 
       clear_heartbeat
@@ -83,7 +84,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.pipelined do
           conn.srem("processes", identity)
-          conn.del("#{identity}:workers")
+          conn.unlink("#{identity}:workers")
         end
       end
     rescue
@@ -96,6 +97,32 @@ module Sidekiq
       ❤
     end
 
+    def self.flush_stats
+      fails = Processor::FAILURE.reset
+      procd = Processor::PROCESSED.reset
+      return if fails + procd == 0
+
+      nowdate = Time.now.utc.strftime("%Y-%m-%d")
+      begin
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.incrby("stat:processed", procd)
+            conn.incrby("stat:processed:#{nowdate}", procd)
+            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            conn.incrby("stat:failed", fails)
+            conn.incrby("stat:failed:#{nowdate}", fails)
+            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+          end
+        end
+      rescue => ex
+        # we're exiting the process, things might be shut down so don't
+        # try to handle the exception
+        Sidekiq.logger.warn("Unable to flush stats: #{ex}")
+      end
+    end
+    at_exit(&method(:flush_stats))
+
     def ❤
       key = identity
       fails = procd = 0
@@ -118,7 +145,7 @@ module Sidekiq
             conn.incrby("stat:failed:#{nowdate}", fails)
             conn.expire("stat:failed:#{nowdate}", STATS_TTL)
 
-            conn.del(workers_key)
+            conn.unlink(workers_key)
             curstate.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
@@ -131,7 +158,7 @@ module Sidekiq
         _, exists, _, _, msg = Sidekiq.redis { |conn|
           conn.multi {
             conn.sadd("processes", key)
-            conn.exists(key)
+            conn.exists?(key)
             conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
@@ -146,7 +173,7 @@ module Sidekiq
         ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
-        logger.error("heartbeat: #{e.message}")
+        logger.error("heartbeat: #{e}")
         # don't lose the counts if there was a network issue
         Processor::PROCESSED.incr(procd)
         Processor::FAILURE.incr(fails)
@@ -163,7 +190,7 @@ module Sidekiq
           "concurrency" => @options[:concurrency],
           "queues" => @options[:queues].uniq,
           "labels" => @options[:labels],
-          "identity" => identity,
+          "identity" => identity
         }
       end
     end

data/lib/sidekiq/logger.rb

@@ -23,7 +23,7 @@ module Sidekiq
       "info" => 1,
       "warn" => 2,
       "error" => 3,
-      "fatal" => 4,
+      "fatal" => 4
     }
     LEVELS.default_proc = proc do |_, level|
       Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
@@ -31,23 +31,23 @@ module Sidekiq
     end
 
     def debug?
-      level >= 0
+      level <= 0
     end
 
     def info?
-      level >= 1
+      level <= 1
     end
 
     def warn?
-      level >= 2
+      level <= 2
     end
 
     def error?
-      level >= 3
+      level <= 3
     end
 
     def fatal?
-      level >= 4
+      level <= 4
     end
 
     def local_level
@@ -83,7 +83,7 @@ module Sidekiq
     # Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
     # FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
     def add(severity, message = nil, progname = nil, &block)
-      severity ||= UNKNOWN
+      severity ||= ::Logger::UNKNOWN
       progname ||= @progname
 
       return true if @logdev.nil? || severity < level
@@ -104,7 +104,7 @@ module Sidekiq
   class Logger < ::Logger
     include LoggingUtils
 
-    def initialize(*args)
+    def initialize(*args, **kwargs)
       super
       self.formatter = Sidekiq.log_formatter
     end
@@ -152,7 +152,7 @@ module Sidekiq
             pid: ::Process.pid,
             tid: tid,
             lvl: severity,
-            msg: message,
+            msg: message
           }
           c = ctx
           hash["ctx"] = c unless c.empty?

data/lib/sidekiq/manager.rb

@@ -35,7 +35,7 @@ module Sidekiq
       @done = false
       @workers = Set.new
       @count.times do
-        @workers << Processor.new(self)
+        @workers << Processor.new(self, options)
       end
       @plock = Mutex.new
     end
@@ -90,7 +90,7 @@ module Sidekiq
       @plock.synchronize do
         @workers.delete(processor)
         unless @done
-          p = Processor.new(self)
+          p = Processor.new(self, options)
           @workers << p
           p.start
         end
@@ -123,7 +123,7 @@ module Sidekiq
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+        strategy = @options[:fetch]
         strategy.bulk_requeue(jobs, @options)
       end
 

data/lib/sidekiq/monitor.rb

@@ -62,7 +62,7 @@ class Sidekiq::Monitor
       columns = {
         name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
         size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
-        latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
       }
       columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
       puts
@@ -101,7 +101,7 @@ class Sidekiq::Monitor
       tags = [
         process["tag"],
         process["labels"],
-        (process["quiet"] == "true" ? "quiet" : nil),
+        (process["quiet"] == "true" ? "quiet" : nil)
       ].flatten.compact
       tags.any? ? "[#{tags.join("] [")}]" : nil
     end

data/lib/sidekiq/processor.rb

@@ -28,15 +28,15 @@ module Sidekiq
     attr_reader :thread
     attr_reader :job
 
-    def initialize(mgr)
+    def initialize(mgr, options)
       @mgr = mgr
       @down = false
       @done = false
       @job = nil
       @thread = nil
-      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
-      @reloader = Sidekiq.options[:reloader]
-      @job_logger = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
+      @strategy = options[:fetch]
+      @reloader = options[:reloader] || proc { |&block| block.call }
+      @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
       @retrier = Sidekiq::JobRetry.new
     end
 
@@ -181,7 +181,7 @@ module Sidekiq
         # the retry subsystem (e.g. network partition).  We won't acknowledge the job
         # so it can be rescued when using Sidekiq Pro.
         handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
-        raise e
+        raise ex
       ensure
         if ack
           # We don't want a shutdown signal to interrupt job acknowledgment.

data/lib/sidekiq/rails.rb

@@ -4,6 +4,22 @@ require "sidekiq/worker"
 
 module Sidekiq
   class Rails < ::Rails::Engine
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
+
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
     # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
@@ -23,8 +39,6 @@ module Sidekiq
 
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
-    # We have to add the reloader after initialize to see if cache_classes has
-    # been turned on.
     #
     # None of this matters on the client-side, only within the Sidekiq process itself.
     config.after_initialize do
@@ -32,21 +46,5 @@ module Sidekiq
         Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
       end
     end
-
-    class Reloader
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        @app.reloader.wrap do
-          yield
-        end
-      end
-
-      def inspect
-        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
-      end
-    end
   end
 end

data/lib/sidekiq/redis_connection.rb

@@ -8,15 +8,14 @@ module Sidekiq
   class RedisConnection
     class << self
       def create(options = {})
-        options.keys.each do |key|
-          options[key.to_sym] = options.delete(key)
-        end
+        symbolized_options = options.transform_keys(&:to_sym)
 
-        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{::Process.pid}" unless options.key?(:id)
-        options[:url] ||= determine_redis_provider
+        if !symbolized_options[:url] && (u = determine_redis_provider)
+          symbolized_options[:url] = u
+        end
 
-        size = if options[:size]
-          options[:size]
+        size = if symbolized_options[:size]
+          symbolized_options[:size]
         elsif Sidekiq.server?
           # Give ourselves plenty of connections.  pool is lazy
           # so we won't create them until we need them.
@@ -29,11 +28,11 @@ module Sidekiq
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
-        pool_timeout = options[:pool_timeout] || 1
-        log_info(options)
+        pool_timeout = symbolized_options[:pool_timeout] || 1
+        log_info(symbolized_options)
 
         ConnectionPool.new(timeout: pool_timeout, size: size) do
-          build_client(options)
+          build_client(symbolized_options)
         end
       end
 
@@ -93,9 +92,15 @@ module Sidekiq
       end
 
       def log_info(options)
-        # Don't log Redis AUTH password
         redacted = "REDACTED"
-        scrubbed_options = options.dup
+
+        # deep clone so we can muck with these options all we want
+        #
+        # exclude SSL params from dump-and-load because some information isn't
+        # safely dumpable in current Rubies
+        keys = options.keys
+        keys.delete(:ssl_params)
+        scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
@@ -103,6 +108,9 @@ module Sidekiq
         if scrubbed_options[:password]
           scrubbed_options[:password] = redacted
         end
+        scrubbed_options[:sentinels]&.each do |sentinel|
+          sentinel[:password] = redacted if sentinel[:password]
+        end
         if Sidekiq.server?
           Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
         else
@@ -119,7 +127,7 @@ module Sidekiq
         # initialization code at all.
         #
         p = ENV["REDIS_PROVIDER"]
-        if p && p =~ /\:/
+        if p && p =~ /:/
           raise <<~EOM
             REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
             Platforms like Heroku will sell addons that publish a *_URL variable.  You need to tell Sidekiq with REDIS_PROVIDER, e.g.:

data/lib/sidekiq/sd_notify.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+# The MIT License
+#
+# Copyright (c) 2017, 2018, 2019, 2020 Agis Anastasopoulos
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# This is a copy of https://github.com/agis/ruby-sdnotify as of commit a7d52ee
+# The only changes made was "rehoming" it within the Sidekiq module to avoid
+# namespace collisions and applying standard's code formatting style.
+
+require "socket"
+
+# SdNotify is a pure-Ruby implementation of sd_notify(3). It can be used to
+# notify systemd about state changes. Methods of this package are no-op on
+# non-systemd systems (eg. Darwin).
+#
+# The API maps closely to the original implementation of sd_notify(3),
+# therefore be sure to check the official man pages prior to using SdNotify.
+#
+# @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+module Sidekiq
+  module SdNotify
+    # Exception raised when there's an error writing to the notification socket
+    class NotifyError < RuntimeError; end
+
+    READY = "READY=1"
+    RELOADING = "RELOADING=1"
+    STOPPING = "STOPPING=1"
+    STATUS = "STATUS="
+    ERRNO = "ERRNO="
+    MAINPID = "MAINPID="
+    WATCHDOG = "WATCHDOG=1"
+    FDSTORE = "FDSTORE=1"
+
+    def self.ready(unset_env = false)
+      notify(READY, unset_env)
+    end
+
+    def self.reloading(unset_env = false)
+      notify(RELOADING, unset_env)
+    end
+
+    def self.stopping(unset_env = false)
+      notify(STOPPING, unset_env)
+    end
+
+    # @param status [String] a custom status string that describes the current
+    #   state of the service
+    def self.status(status, unset_env = false)
+      notify("#{STATUS}#{status}", unset_env)
+    end
+
+    # @param errno [Integer]
+    def self.errno(errno, unset_env = false)
+      notify("#{ERRNO}#{errno}", unset_env)
+    end
+
+    # @param pid [Integer]
+    def self.mainpid(pid, unset_env = false)
+      notify("#{MAINPID}#{pid}", unset_env)
+    end
+
+    def self.watchdog(unset_env = false)
+      notify(WATCHDOG, unset_env)
+    end
+
+    def self.fdstore(unset_env = false)
+      notify(FDSTORE, unset_env)
+    end
+
+    # @return [Boolean] true if the service manager expects watchdog keep-alive
+    #   notification messages to be sent from this process.
+    #
+    # If the $WATCHDOG_USEC environment variable is set,
+    # and the $WATCHDOG_PID variable is unset or set to the PID of the current
+    # process
+    #
+    # @note Unlike sd_watchdog_enabled(3), this method does not mutate the
+    #   environment.
+    def self.watchdog?
+      wd_usec = ENV["WATCHDOG_USEC"]
+      wd_pid = ENV["WATCHDOG_PID"]
+
+      return false unless wd_usec
+
+      begin
+        wd_usec = Integer(wd_usec)
+      rescue
+        return false
+      end
+
+      return false if wd_usec <= 0
+      return true if !wd_pid || wd_pid == $$.to_s
+
+      false
+    end
+
+    # Notify systemd with the provided state, via the notification socket, if
+    # any.
+    #
+    # Generally this method will be used indirectly through the other methods
+    # of the library.
+    #
+    # @param state [String]
+    # @param unset_env [Boolean]
+    #
+    # @return [Fixnum, nil] the number of bytes written to the notification
+    #   socket or nil if there was no socket to report to (eg. the program wasn't
+    #   started by systemd)
+    #
+    # @raise [NotifyError] if there was an error communicating with the systemd
+    #   socket
+    #
+    # @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+    def self.notify(state, unset_env = false)
+      sock = ENV["NOTIFY_SOCKET"]
+
+      return nil unless sock
+
+      ENV.delete("NOTIFY_SOCKET") if unset_env
+
+      begin
+        Addrinfo.unix(sock, :DGRAM).connect do |s|
+          s.close_on_exec = true
+          s.write(state)
+        end
+      rescue => e
+        raise NotifyError, "#{e.class}: #{e.message}", e.backtrace
+      end
+    end
+  end
+end