sidekiq 6.0.0 → 6.3.0

data/lib/sidekiq/processor.rb

@@ -28,15 +28,15 @@ module Sidekiq
     attr_reader :thread
     attr_reader :job
 
-    def initialize(mgr)
+    def initialize(mgr, options)
       @mgr = mgr
       @down = false
       @done = false
       @job = nil
       @thread = nil
-      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
-      @reloader = Sidekiq.options[:reloader]
-      @job_logger = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
+      @strategy = options[:fetch]
+      @reloader = options[:reloader] || proc { |&block| block.call }
+      @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
       @retrier = Sidekiq::JobRetry.new
     end
 
@@ -111,16 +111,19 @@ module Sidekiq
       nil
     end
 
-    def dispatch(job_hash, queue)
+    def dispatch(job_hash, queue, jobstr)
       # since middleware can mutate the job hash
-      # we clone here so we report the original
+      # we need to clone it to report the original
       # job structure to the Web UI
-      pristine = cloned(job_hash)
+      # or to push back to redis when retrying.
+      # To avoid costly and, most of the time, useless cloning here,
+      # we pass original String of JSON to respected methods
+      # to re-parse it there if we need access to the original, untouched job
 
-      @job_logger.with_job_hash_context(job_hash) do
-        @retrier.global(pristine, queue) do
+      @job_logger.prepare(job_hash) do
+        @retrier.global(jobstr, queue) do
           @job_logger.call(job_hash, queue) do
-            stats(pristine, queue) do
+            stats(jobstr, queue) do
               # Rails 5 requires a Reloader to wrap code execution.  In order to
               # constantize the worker and instantiate an instance, we have to call
               # the Reloader.  It handles code loading, db connection management, etc.
@@ -129,7 +132,7 @@ module Sidekiq
                 klass = constantize(job_hash["class"])
                 worker = klass.new
                 worker.jid = job_hash["jid"]
-                @retrier.local(worker, pristine, queue) do
+                @retrier.local(worker, jobstr, queue) do
                   yield worker
                 end
               end
@@ -156,9 +159,9 @@ module Sidekiq
 
       ack = false
       begin
-        dispatch(job_hash, queue) do |worker|
+        dispatch(job_hash, queue, jobstr) do |worker|
           Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
-            execute_job(worker, cloned(job_hash["args"]))
+            execute_job(worker, job_hash["args"])
           end
         end
         ack = true
@@ -178,7 +181,7 @@ module Sidekiq
         # the retry subsystem (e.g. network partition).  We won't acknowledge the job
         # so it can be rescued when using Sidekiq Pro.
         handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
-        raise e
+        raise ex
       ensure
         if ack
           # We don't want a shutdown signal to interrupt job acknowledgment.
@@ -247,8 +250,8 @@ module Sidekiq
     FAILURE = Counter.new
     WORKER_STATE = SharedWorkerState.new
 
-    def stats(job_hash, queue)
-      WORKER_STATE.set(tid, {queue: queue, payload: job_hash, run_at: Time.now.to_i})
+    def stats(jobstr, queue)
+      WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
 
       begin
         yield
@@ -261,21 +264,16 @@ module Sidekiq
       end
     end
 
-    # Deep clone the arguments passed to the worker so that if
-    # the job fails, what is pushed back onto Redis hasn't
-    # been mutated by the worker.
-    def cloned(thing)
-      Marshal.load(Marshal.dump(thing))
-    end
-
     def constantize(str)
+      return Object.const_get(str) unless str.include?("::")
+
       names = str.split("::")
       names.shift if names.empty? || names.first.empty?
 
       names.inject(Object) do |constant, name|
         # the false flag limits search for name to under the constant namespace
         #   which mimics Rails' behaviour
-        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
+        constant.const_get(name, false)
       end
     end
   end

data/lib/sidekiq/rails.rb

@@ -4,6 +4,22 @@ require "sidekiq/worker"
 
 module Sidekiq
   class Rails < ::Rails::Engine
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
+
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
     # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
@@ -21,10 +37,19 @@ module Sidekiq
       end
     end
 
+    initializer "sidekiq.rails_logger" do
+      Sidekiq.configure_server do |_|
+        # This is the integration code necessary so that if code uses `Rails.logger.info "Hello"`,
+        # it will appear in the Sidekiq console with all of the job context. See #5021 and
+        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
+        unless ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
+        end
+      end
+    end
+
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
-    # We have to add the reloader after initialize to see if cache_classes has
-    # been turned on.
     #
     # None of this matters on the client-side, only within the Sidekiq process itself.
     config.after_initialize do
@@ -32,21 +57,5 @@ module Sidekiq
         Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
       end
     end
-
-    class Reloader
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        @app.reloader.wrap do
-          yield
-        end
-      end
-
-      def inspect
-        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
-      end
-    end
   end
 end

data/lib/sidekiq/redis_connection.rb

@@ -8,15 +8,14 @@ module Sidekiq
   class RedisConnection
     class << self
       def create(options = {})
-        options.keys.each do |key|
-          options[key.to_sym] = options.delete(key)
-        end
+        symbolized_options = options.transform_keys(&:to_sym)
 
-        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{::Process.pid}" unless options.key?(:id)
-        options[:url] ||= determine_redis_provider
+        if !symbolized_options[:url] && (u = determine_redis_provider)
+          symbolized_options[:url] = u
+        end
 
-        size = if options[:size]
-          options[:size]
+        size = if symbolized_options[:size]
+          symbolized_options[:size]
         elsif Sidekiq.server?
           # Give ourselves plenty of connections.  pool is lazy
           # so we won't create them until we need them.
@@ -29,11 +28,11 @@ module Sidekiq
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
-        pool_timeout = options[:pool_timeout] || 1
-        log_info(options)
+        pool_timeout = symbolized_options[:pool_timeout] || 1
+        log_info(symbolized_options)
 
         ConnectionPool.new(timeout: pool_timeout, size: size) do
-          build_client(options)
+          build_client(symbolized_options)
         end
       end
 
@@ -93,9 +92,13 @@ module Sidekiq
       end
 
       def log_info(options)
-        # Don't log Redis AUTH password
         redacted = "REDACTED"
-        scrubbed_options = options.dup
+
+        # Deep clone so we can muck with these options all we want and exclude
+        # params from dump-and-load that may contain objects that Marshal is
+        # unable to safely dump.
+        keys = options.keys - [:logger, :ssl_params]
+        scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
@@ -103,6 +106,9 @@ module Sidekiq
         if scrubbed_options[:password]
           scrubbed_options[:password] = redacted
         end
+        scrubbed_options[:sentinels]&.each do |sentinel|
+          sentinel[:password] = redacted if sentinel[:password]
+        end
         if Sidekiq.server?
           Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
         else
@@ -119,7 +125,7 @@ module Sidekiq
         # initialization code at all.
         #
         p = ENV["REDIS_PROVIDER"]
-        if p && p =~ /\:/
+        if p && p =~ /:/
           raise <<~EOM
             REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
             Platforms like Heroku will sell addons that publish a *_URL variable.  You need to tell Sidekiq with REDIS_PROVIDER, e.g.:

data/lib/sidekiq/scheduled.rb

@@ -9,28 +9,48 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
+      LUA_ZPOPBYSCORE = <<~LUA
+        local key, now = KEYS[1], ARGV[1]
+        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        if jobs[1] then
+          redis.call("zrem", key, jobs[1])
+          return jobs[1]
+        end
+      LUA
+
+      def initialize
+        @lua_zpopbyscore_sha = nil
+      end
+
       def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get the next item in the queue if it's score (time to execute) is <= now.
+            # Get next item in the queue with score (time to execute) <= now.
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
-            while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first)
-
-              # Pop item off the queue and add it to the work queue. If the job can't be popped from
-              # the queue, it's because another process already popped it so we can move on to the
-              # next one.
-              if conn.zrem(sorted_set, job)
-                Sidekiq::Client.push(Sidekiq.load_json(job))
-                Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
-              end
+            while (job = zpopbyscore(conn, keys: [sorted_set], argv: [now]))
+              Sidekiq::Client.push(Sidekiq.load_json(job))
+              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
       end
+
+      private
+
+      def zpopbyscore(conn, keys: nil, argv: nil)
+        @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE) if @lua_zpopbyscore_sha.nil?
+
+        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?("NOSCRIPT")
+
+        @lua_zpopbyscore_sha = nil
+        retry
+      end
     end
 
     ##
@@ -48,6 +68,7 @@ module Sidekiq
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
+        @count_calls = 0
       end
 
       # Shut down this instance, will pause until the thread is dead.
@@ -151,8 +172,13 @@ module Sidekiq
       end
 
       def process_count
-        pcount = Sidekiq::ProcessSet.new.size
+        # The work buried within Sidekiq::ProcessSet#cleanup can be
+        # expensive at scale. Cut it down by 90% with this counter.
+        # NB: This method is only called by the scheduler thread so we
+        # don't need to worry about the thread safety of +=.
+        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
         pcount = 1 if pcount == 0
+        @count_calls += 1
         pcount
       end
 

data/lib/sidekiq/sd_notify.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+# The MIT License
+#
+# Copyright (c) 2017, 2018, 2019, 2020 Agis Anastasopoulos
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# This is a copy of https://github.com/agis/ruby-sdnotify as of commit a7d52ee
+# The only changes made was "rehoming" it within the Sidekiq module to avoid
+# namespace collisions and applying standard's code formatting style.
+
+require "socket"
+
+# SdNotify is a pure-Ruby implementation of sd_notify(3). It can be used to
+# notify systemd about state changes. Methods of this package are no-op on
+# non-systemd systems (eg. Darwin).
+#
+# The API maps closely to the original implementation of sd_notify(3),
+# therefore be sure to check the official man pages prior to using SdNotify.
+#
+# @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+module Sidekiq
+  module SdNotify
+    # Exception raised when there's an error writing to the notification socket
+    class NotifyError < RuntimeError; end
+
+    READY = "READY=1"
+    RELOADING = "RELOADING=1"
+    STOPPING = "STOPPING=1"
+    STATUS = "STATUS="
+    ERRNO = "ERRNO="
+    MAINPID = "MAINPID="
+    WATCHDOG = "WATCHDOG=1"
+    FDSTORE = "FDSTORE=1"
+
+    def self.ready(unset_env = false)
+      notify(READY, unset_env)
+    end
+
+    def self.reloading(unset_env = false)
+      notify(RELOADING, unset_env)
+    end
+
+    def self.stopping(unset_env = false)
+      notify(STOPPING, unset_env)
+    end
+
+    # @param status [String] a custom status string that describes the current
+    #   state of the service
+    def self.status(status, unset_env = false)
+      notify("#{STATUS}#{status}", unset_env)
+    end
+
+    # @param errno [Integer]
+    def self.errno(errno, unset_env = false)
+      notify("#{ERRNO}#{errno}", unset_env)
+    end
+
+    # @param pid [Integer]
+    def self.mainpid(pid, unset_env = false)
+      notify("#{MAINPID}#{pid}", unset_env)
+    end
+
+    def self.watchdog(unset_env = false)
+      notify(WATCHDOG, unset_env)
+    end
+
+    def self.fdstore(unset_env = false)
+      notify(FDSTORE, unset_env)
+    end
+
+    # @return [Boolean] true if the service manager expects watchdog keep-alive
+    #   notification messages to be sent from this process.
+    #
+    # If the $WATCHDOG_USEC environment variable is set,
+    # and the $WATCHDOG_PID variable is unset or set to the PID of the current
+    # process
+    #
+    # @note Unlike sd_watchdog_enabled(3), this method does not mutate the
+    #   environment.
+    def self.watchdog?
+      wd_usec = ENV["WATCHDOG_USEC"]
+      wd_pid = ENV["WATCHDOG_PID"]
+
+      return false unless wd_usec
+
+      begin
+        wd_usec = Integer(wd_usec)
+      rescue
+        return false
+      end
+
+      return false if wd_usec <= 0
+      return true if !wd_pid || wd_pid == $$.to_s
+
+      false
+    end
+
+    # Notify systemd with the provided state, via the notification socket, if
+    # any.
+    #
+    # Generally this method will be used indirectly through the other methods
+    # of the library.
+    #
+    # @param state [String]
+    # @param unset_env [Boolean]
+    #
+    # @return [Fixnum, nil] the number of bytes written to the notification
+    #   socket or nil if there was no socket to report to (eg. the program wasn't
+    #   started by systemd)
+    #
+    # @raise [NotifyError] if there was an error communicating with the systemd
+    #   socket
+    #
+    # @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+    def self.notify(state, unset_env = false)
+      sock = ENV["NOTIFY_SOCKET"]
+
+      return nil unless sock
+
+      ENV.delete("NOTIFY_SOCKET") if unset_env
+
+      begin
+        Addrinfo.unix(sock, :DGRAM).connect do |s|
+          s.close_on_exec = true
+          s.write(state)
+        end
+      rescue => e
+        raise NotifyError, "#{e.class}: #{e.message}", e.backtrace
+      end
+    end
+  end
+end

data/lib/sidekiq/systemd.rb

@@ -0,0 +1,24 @@
+#
+# Sidekiq's systemd integration allows Sidekiq to inform systemd:
+#  1. when it has successfully started
+#  2. when it is starting shutdown
+#  3. periodically for a liveness check with a watchdog thread
+#
+module Sidekiq
+  def self.start_watchdog
+    usec = Integer(ENV["WATCHDOG_USEC"])
+    return Sidekiq.logger.error("systemd Watchdog too fast: " + usec) if usec < 1_000_000
+
+    sec_f = usec / 1_000_000.0
+    # "It is recommended that a daemon sends a keep-alive notification message
+    # to the service manager every half of the time returned here."
+    ping_f = sec_f / 2
+    Sidekiq.logger.info "Pinging systemd watchdog every #{ping_f.round(1)} sec"
+    Thread.new do
+      loop do
+        sleep ping_f
+        Sidekiq::SdNotify.watchdog
+      end
+    end
+  end
+end

data/lib/sidekiq/testing.rb

@@ -323,10 +323,20 @@ module Sidekiq
       end
     end
   end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+      end
+    end
+  end
+
+  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
+  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
-if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
-  puts("**************************************************")
-  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
-  puts("**************************************************")
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
+  warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end

data/lib/sidekiq/util.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "forwardable"
 require "socket"
 require "securerandom"
 require "sidekiq/exception_handler"
@@ -8,11 +9,36 @@ module Sidekiq
   ##
   # This module is part of Sidekiq core and not intended for extensions.
   #
+
+  class RingBuffer
+    include Enumerable
+    extend Forwardable
+    def_delegators :@buf, :[], :each, :size
+
+    def initialize(size, default = 0)
+      @size = size
+      @buf = Array.new(size, default)
+      @index = 0
+    end
+
+    def <<(element)
+      @buf[@index % @size] = element
+      @index += 1
+      element
+    end
+
+    def buffer
+      @buf
+    end
+
+    def reset(default = 0)
+      @buf.fill(default)
+    end
+  end
+
   module Util
     include ExceptionHandler
 
-    EXPIRY = 60 * 60 * 24
-
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.0.0"
+  VERSION = "6.3.0"
 end

data/lib/sidekiq/web/action.rb

@@ -15,7 +15,7 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, res
+      throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, [Sidekiq.dump_json(payload)]]
+      [200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)