sidekiq 5.2.9 → 6.4.0

data/lib/sidekiq/redis_connection.rb

@@ -1,37 +1,38 @@
 # frozen_string_literal: true
-require 'connection_pool'
-require 'redis'
-require 'uri'
+
+require "connection_pool"
+require "redis"
+require "uri"
 
 module Sidekiq
   class RedisConnection
     class << self
+      def create(options = {})
+        symbolized_options = options.transform_keys(&:to_sym)
 
-      def create(options={})
-        options.keys.each do |key|
-          options[key.to_sym] = options.delete(key)
+        if !symbolized_options[:url] && (u = determine_redis_provider)
+          symbolized_options[:url] = u
         end
 
-        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{$$}" if !options.has_key?(:id)
-        options[:url] ||= determine_redis_provider
-
-        size = if options[:size]
-                 options[:size]
-               elsif Sidekiq.server?
-                 Sidekiq.options[:concurrency] + 5
-               elsif ENV['RAILS_MAX_THREADS']
-                 Integer(ENV['RAILS_MAX_THREADS'])
-               else
-                 5
-               end
+        size = if symbolized_options[:size]
+          symbolized_options[:size]
+        elsif Sidekiq.server?
+          # Give ourselves plenty of connections.  pool is lazy
+          # so we won't create them until we need them.
+          Sidekiq.options[:concurrency] + 5
+        elsif ENV["RAILS_MAX_THREADS"]
+          Integer(ENV["RAILS_MAX_THREADS"])
+        else
+          5
+        end
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
-        pool_timeout = options[:pool_timeout] || 1
-        log_info(options)
+        pool_timeout = symbolized_options[:pool_timeout] || 1
+        log_info(symbolized_options)
 
-        ConnectionPool.new(:timeout => pool_timeout, :size => size) do
-          build_client(options)
+        ConnectionPool.new(timeout: pool_timeout, size: size) do
+          build_client(symbolized_options)
         end
       end
 
@@ -46,7 +47,7 @@ module Sidekiq
       #   - enterprise's leader election
       #   - enterprise's cron support
       def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size <= concurrency
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
       end
 
       def build_client(options)
@@ -55,8 +56,8 @@ module Sidekiq
         client = Redis.new client_opts(options)
         if namespace
           begin
-            require 'redis/namespace'
-            Redis::Namespace.new(namespace, :redis => client)
+            require "redis/namespace"
+            Redis::Namespace.new(namespace, redis: client)
           rescue LoadError
             Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
                                  "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
@@ -78,7 +79,7 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] ||= Redis::Connection.drivers.last || 'ruby'
+        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
 
         # Issue #3303, redis-rb will silently retry an operation.
         # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
@@ -91,9 +92,13 @@ module Sidekiq
       end
 
       def log_info(options)
-        # Don't log Redis AUTH password
         redacted = "REDACTED"
-        scrubbed_options = options.dup
+
+        # Deep clone so we can muck with these options all we want and exclude
+        # params from dump-and-load that may contain objects that Marshal is
+        # unable to safely dump.
+        keys = options.keys - [:logger, :ssl_params]
+        scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
@@ -101,6 +106,9 @@ module Sidekiq
         if scrubbed_options[:password]
           scrubbed_options[:password] = redacted
         end
+        scrubbed_options[:sentinels]&.each do |sentinel|
+          sentinel[:password] = redacted if sentinel[:password]
+        end
         if Sidekiq.server?
           Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
         else
@@ -115,30 +123,22 @@ module Sidekiq
         # REDIS_PROVIDER=MY_REDIS_URL
         # and Sidekiq will find your custom URL variable with no custom
         # initialization code at all.
-        p = ENV['REDIS_PROVIDER']
-        if p && p =~ /\:/
-          Sidekiq.logger.error <<-EOM
-
-#################################################################################
-
-REDIS_PROVIDER should be set to the **name** of the variable which contains the Redis URL, not a URL itself.
-Platforms like Heroku sell addons that publish a *_URL variable.  You tell Sidekiq with REDIS_PROVIDER, e.g.:
-
-  REDIS_PROVIDER=REDISTOGO_URL
-  REDISTOGO_URL=redis://somehost.example.com:6379/4
-
-Use REDIS_URL if you wish to point Sidekiq to a URL directly.
-
-This configuration error will crash starting in Sidekiq 5.3.
-
-#################################################################################
-EOM
+        #
+        p = ENV["REDIS_PROVIDER"]
+        if p && p =~ /:/
+          raise <<~EOM
+            REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
+            Platforms like Heroku will sell addons that publish a *_URL variable.  You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
+
+            REDISTOGO_URL=redis://somehost.example.com:6379/4
+            REDIS_PROVIDER=REDISTOGO_URL
+          EOM
         end
+
         ENV[
-          ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+          p || "REDIS_URL"
         ]
       end
-
     end
   end
 end

data/lib/sidekiq/scheduled.rb

@@ -1,35 +1,64 @@
 # frozen_string_literal: true
-require 'sidekiq'
-require 'sidekiq/util'
-require 'sidekiq/api'
+
+require "sidekiq"
+require "sidekiq/util"
+require "sidekiq/api"
 
 module Sidekiq
   module Scheduled
-    SETS = %w(retry schedule)
+    SETS = %w[retry schedule]
 
     class Enq
-      def enqueue_jobs(now=Time.now.to_f.to_s, sorted_sets=SETS)
+      LUA_ZPOPBYSCORE = <<~LUA
+        local key, now = KEYS[1], ARGV[1]
+        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        if jobs[1] then
+          redis.call("zrem", key, jobs[1])
+          return jobs[1]
+        end
+      LUA
+
+      def initialize
+        @done = false
+        @lua_zpopbyscore_sha = nil
+      end
+
+      def enqueue_jobs(sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get the next item in the queue if it's score (time to execute) is <= now.
+            # Get next item in the queue with score (time to execute) <= now.
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
-            while job = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
-
-              # Pop item off the queue and add it to the work queue. If the job can't be popped from
-              # the queue, it's because another process already popped it so we can move on to the
-              # next one.
-              if conn.zrem(sorted_set, job)
-                Sidekiq::Client.push(Sidekiq.load_json(job))
-                Sidekiq::Logging.logger.debug { "enqueued #{sorted_set}: #{job}" }
-              end
+            while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
+              Sidekiq::Client.push(Sidekiq.load_json(job))
+              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
       end
+
+      def terminate
+        @done = true
+      end
+
+      private
+
+      def zpopbyscore(conn, keys: nil, argv: nil)
+        if @lua_zpopbyscore_sha.nil?
+          raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
+          @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
+        end
+
+        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?("NOSCRIPT")
+
+        @lua_zpopbyscore_sha = nil
+        retry
+      end
     end
 
     ##
@@ -47,11 +76,14 @@ module Sidekiq
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
+        @count_calls = 0
       end
 
       # Shut down this instance, will pause until the thread is dead.
       def terminate
         @done = true
+        @enq.terminate if @enq.respond_to?(:terminate)
+
         if @thread
           t = @thread
           @thread = nil
@@ -61,26 +93,24 @@ module Sidekiq
       end
 
       def start
-        @thread ||= safe_thread("scheduler") do
+        @thread ||= safe_thread("scheduler") {
           initial_wait
 
-          while !@done
+          until @done
             enqueue
             wait
           end
           Sidekiq.logger.info("Scheduler exiting...")
-        end
+        }
       end
 
       def enqueue
-        begin
-          @enq.enqueue_jobs
-        rescue => ex
-          # Most likely a problem with redis networking.
-          # Punt and try again at the next interval
-          logger.error ex.message
-          handle_exception(ex)
-        end
+        @enq.enqueue_jobs
+      rescue => ex
+        # Most likely a problem with redis networking.
+        # Punt and try again at the next interval
+        logger.error ex.message
+        handle_exception(ex)
       end
 
       private
@@ -152,8 +182,13 @@ module Sidekiq
       end
 
       def process_count
-        pcount = Sidekiq::ProcessSet.new.size
+        # The work buried within Sidekiq::ProcessSet#cleanup can be
+        # expensive at scale. Cut it down by 90% with this counter.
+        # NB: This method is only called by the scheduler thread so we
+        # don't need to worry about the thread safety of +=.
+        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
         pcount = 1 if pcount == 0
+        @count_calls += 1
         pcount
       end
 
@@ -168,7 +203,6 @@ module Sidekiq
         @sleeper.pop(total)
       rescue Timeout::Error
       end
-
     end
   end
 end

data/lib/sidekiq/sd_notify.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+# The MIT License
+#
+# Copyright (c) 2017, 2018, 2019, 2020 Agis Anastasopoulos
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# This is a copy of https://github.com/agis/ruby-sdnotify as of commit a7d52ee
+# The only changes made was "rehoming" it within the Sidekiq module to avoid
+# namespace collisions and applying standard's code formatting style.
+
+require "socket"
+
+# SdNotify is a pure-Ruby implementation of sd_notify(3). It can be used to
+# notify systemd about state changes. Methods of this package are no-op on
+# non-systemd systems (eg. Darwin).
+#
+# The API maps closely to the original implementation of sd_notify(3),
+# therefore be sure to check the official man pages prior to using SdNotify.
+#
+# @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+module Sidekiq
+  module SdNotify
+    # Exception raised when there's an error writing to the notification socket
+    class NotifyError < RuntimeError; end
+
+    READY = "READY=1"
+    RELOADING = "RELOADING=1"
+    STOPPING = "STOPPING=1"
+    STATUS = "STATUS="
+    ERRNO = "ERRNO="
+    MAINPID = "MAINPID="
+    WATCHDOG = "WATCHDOG=1"
+    FDSTORE = "FDSTORE=1"
+
+    def self.ready(unset_env = false)
+      notify(READY, unset_env)
+    end
+
+    def self.reloading(unset_env = false)
+      notify(RELOADING, unset_env)
+    end
+
+    def self.stopping(unset_env = false)
+      notify(STOPPING, unset_env)
+    end
+
+    # @param status [String] a custom status string that describes the current
+    #   state of the service
+    def self.status(status, unset_env = false)
+      notify("#{STATUS}#{status}", unset_env)
+    end
+
+    # @param errno [Integer]
+    def self.errno(errno, unset_env = false)
+      notify("#{ERRNO}#{errno}", unset_env)
+    end
+
+    # @param pid [Integer]
+    def self.mainpid(pid, unset_env = false)
+      notify("#{MAINPID}#{pid}", unset_env)
+    end
+
+    def self.watchdog(unset_env = false)
+      notify(WATCHDOG, unset_env)
+    end
+
+    def self.fdstore(unset_env = false)
+      notify(FDSTORE, unset_env)
+    end
+
+    # @return [Boolean] true if the service manager expects watchdog keep-alive
+    #   notification messages to be sent from this process.
+    #
+    # If the $WATCHDOG_USEC environment variable is set,
+    # and the $WATCHDOG_PID variable is unset or set to the PID of the current
+    # process
+    #
+    # @note Unlike sd_watchdog_enabled(3), this method does not mutate the
+    #   environment.
+    def self.watchdog?
+      wd_usec = ENV["WATCHDOG_USEC"]
+      wd_pid = ENV["WATCHDOG_PID"]
+
+      return false unless wd_usec
+
+      begin
+        wd_usec = Integer(wd_usec)
+      rescue
+        return false
+      end
+
+      return false if wd_usec <= 0
+      return true if !wd_pid || wd_pid == $$.to_s
+
+      false
+    end
+
+    # Notify systemd with the provided state, via the notification socket, if
+    # any.
+    #
+    # Generally this method will be used indirectly through the other methods
+    # of the library.
+    #
+    # @param state [String]
+    # @param unset_env [Boolean]
+    #
+    # @return [Fixnum, nil] the number of bytes written to the notification
+    #   socket or nil if there was no socket to report to (eg. the program wasn't
+    #   started by systemd)
+    #
+    # @raise [NotifyError] if there was an error communicating with the systemd
+    #   socket
+    #
+    # @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+    def self.notify(state, unset_env = false)
+      sock = ENV["NOTIFY_SOCKET"]
+
+      return nil unless sock
+
+      ENV.delete("NOTIFY_SOCKET") if unset_env
+
+      begin
+        Addrinfo.unix(sock, :DGRAM).connect do |s|
+          s.close_on_exec = true
+          s.write(state)
+        end
+      rescue => e
+        raise NotifyError, "#{e.class}: #{e.message}", e.backtrace
+      end
+    end
+  end
+end

data/lib/sidekiq/systemd.rb

@@ -0,0 +1,24 @@
+#
+# Sidekiq's systemd integration allows Sidekiq to inform systemd:
+#  1. when it has successfully started
+#  2. when it is starting shutdown
+#  3. periodically for a liveness check with a watchdog thread
+#
+module Sidekiq
+  def self.start_watchdog
+    usec = Integer(ENV["WATCHDOG_USEC"])
+    return Sidekiq.logger.error("systemd Watchdog too fast: " + usec) if usec < 1_000_000
+
+    sec_f = usec / 1_000_000.0
+    # "It is recommended that a daemon sends a keep-alive notification message
+    # to the service manager every half of the time returned here."
+    ping_f = sec_f / 2
+    Sidekiq.logger.info "Pinging systemd watchdog every #{ping_f.round(1)} sec"
+    Thread.new do
+      loop do
+        sleep ping_f
+        Sidekiq::SdNotify.watchdog
+      end
+    end
+  end
+end

data/lib/sidekiq/testing/inline.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
-require 'sidekiq/testing'
+
+require "sidekiq/testing"
 
 ##
 # The Sidekiq inline infrastructure overrides perform_async so that it

data/lib/sidekiq/testing.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
-require 'securerandom'
-require 'sidekiq'
 
-module Sidekiq
+require "securerandom"
+require "sidekiq"
 
+module Sidekiq
   class Testing
     class << self
       attr_accessor :__test_mode
 
       def __set_test_mode(mode)
         if block_given?
-          current_mode = self.__test_mode
+          current_mode = __test_mode
           begin
             self.__test_mode = mode
             yield
@@ -35,19 +35,19 @@ module Sidekiq
       end
 
       def enabled?
-        self.__test_mode != :disable
+        __test_mode != :disable
       end
 
       def disabled?
-        self.__test_mode == :disable
+        __test_mode == :disable
       end
 
       def fake?
-        self.__test_mode == :fake
+        __test_mode == :fake
       end
 
       def inline?
-        self.__test_mode == :inline
+        __test_mode == :inline
       end
 
       def server_middleware
@@ -57,7 +57,7 @@ module Sidekiq
       end
 
       def constantize(str)
-        names = str.split('::')
+        names = str.split("::")
         names.shift if names.empty? || names.first.empty?
 
         names.inject(Object) do |constant, name|
@@ -77,14 +77,14 @@ module Sidekiq
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
-          job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
-          Queues.push(job['queue'], job['class'], job)
+          job["enqueued_at"] = Time.now.to_f unless job["at"]
+          Queues.push(job["queue"], job["class"], job)
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = Sidekiq::Testing.constantize(job['class'])
-          job['id'] ||= SecureRandom.hex(12)
+          klass = Sidekiq::Testing.constantize(job["class"])
+          job["id"] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
         end
@@ -255,27 +255,26 @@ module Sidekiq
     #   Then I should receive a welcome email to "foo@example.com"
     #
     module ClassMethods
-
       # Queue for this worker
       def queue
-        self.sidekiq_options["queue"]
+        get_sidekiq_options["queue"]
       end
 
       # Jobs queued for this worker
       def jobs
-        Queues.jobs_by_worker[self.to_s]
+        Queues.jobs_by_worker[to_s]
       end
 
       # Clear all jobs for this worker
       def clear
-        Queues.clear_for(queue, self.to_s)
+        Queues.clear_for(queue, to_s)
       end
 
       # Drain and run all jobs for this worker
       def drain
         while jobs.any?
           next_job = jobs.first
-          Queues.delete_for(next_job["jid"], next_job["queue"], self.to_s)
+          Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
           process_job(next_job)
         end
       end
@@ -284,16 +283,16 @@ module Sidekiq
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
         next_job = jobs.first
-        Queues.delete_for(next_job["jid"], queue, self.to_s)
+        Queues.delete_for(next_job["jid"], queue, to_s)
         process_job(next_job)
       end
 
       def process_job(job)
         worker = new
-        worker.jid = job['jid']
-        worker.bid = job['bid'] if worker.respond_to?(:bid=)
-        Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
-          execute_job(worker, job['args'])
+        worker.jid = job["jid"]
+        worker.bid = job["bid"] if worker.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
+          execute_job(worker, job["args"])
         end
       end
 
@@ -324,10 +323,20 @@ module Sidekiq
       end
     end
   end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+      end
+    end
+  end
+
+  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
+  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
-if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
-  puts("**************************************************")
-  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
-  puts("**************************************************")
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
+  warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end