sidekiq 4.0.0 → 5.0.0

data/lib/sidekiq/cli.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+# frozen_string_literal: true
 $stdout.sync = true
 
 require 'yaml'
@@ -42,6 +43,10 @@ module Sidekiq
       write_pid
     end
 
+    def jruby?
+      defined?(::JRUBY_VERSION)
+    end
+
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
@@ -50,8 +55,14 @@ module Sidekiq
       print_banner
 
       self_read, self_write = IO.pipe
+      sigs = %w(INT TERM TTIN TSTP)
+      # USR1 and USR2 don't work on the JVM
+      if !jruby?
+        sigs << 'USR1'
+        sigs << 'USR2'
+      end
 
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+      sigs.each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -65,20 +76,20 @@ module Sidekiq
       logger.info Sidekiq::LICENSE
       logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
 
-      Sidekiq.redis do |conn|
-        # touch the connection pool so it is created before we
-        # fire startup and start multithreading.
-        ver = conn.info['redis_version']
-        raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
-      end
+      # touch the connection pool so it is created before we
+      # fire startup and start multithreading.
+      ver = Sidekiq.redis_info['redis_version']
+      raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
+
+      # Touch middleware so it isn't lazy loaded by multiple threads, #3043
+      Sidekiq.server_middleware
 
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
       fire_event(:startup)
 
-      logger.debug {
-        "Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}"
-      }
+      logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
+      logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
 
       if !options[:daemon]
         logger.info 'Starting processing, hit Ctrl-C to stop'
@@ -134,6 +145,10 @@ module Sidekiq
       when 'USR1'
         Sidekiq.logger.info "Received USR1, no longer accepting new work"
         launcher.quiet
+      when 'TSTP'
+        # USR1 is not available on JVM, allow TSTP as an alternate signal
+        Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+        launcher.quiet
       when 'USR2'
         if Sidekiq.options[:logfile]
           Sidekiq.logger.info "Received USR2, reopening log file"
@@ -141,7 +156,7 @@ module Sidekiq
         end
       when 'TTIN'
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+          Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['sidekiq_label']}"
           if thread.backtrace
             Sidekiq.logger.warn thread.backtrace.join("\n")
           else
@@ -207,6 +222,8 @@ module Sidekiq
       opts = parse_config(cfile).merge(opts) if cfile
 
       opts[:strict] = true if opts[:strict].nil?
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
+      opts[:identity] = identity
 
       options.merge!(opts)
     end
@@ -223,21 +240,26 @@ module Sidekiq
       if File.directory?(options[:require])
         require 'rails'
         if ::Rails::VERSION::MAJOR < 4
-          require 'sidekiq/rails'
-          require File.expand_path("#{options[:require]}/config/environment.rb")
-          ::Rails.application.eager_load!
-        else
+          raise "Sidekiq no longer supports this version of Rails"
+        elsif ::Rails::VERSION::MAJOR == 4
           # Painful contortions, see 1791 for discussion
+          # No autoloading, we want to force eager load for everything.
           require File.expand_path("#{options[:require]}/config/application.rb")
           ::Rails::Application.initializer "sidekiq.eager_load" do
             ::Rails.application.config.eager_load = true
           end
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
+        else
+          require 'sidekiq/rails'
+          require File.expand_path("#{options[:require]}/config/environment.rb")
         end
         options[:tag] ||= default_tag
       else
-        require options[:require]
+        not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
+            "./#{options[:require]} or /path/to/#{options[:require]}"
+
+        require(options[:require]) || raise(ArgumentError, not_required_message)
       end
     end
 

data/lib/sidekiq/client.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'securerandom'
 require 'sidekiq/middleware/chain'
 
@@ -35,10 +36,8 @@ module Sidekiq
     #   Sidekiq::Client.new(ConnectionPool.new { Redis.new })
     #
     # Generally this is only needed for very large Sidekiq installs processing
-    # more than thousands jobs per second.  I do not recommend sharding unless
-    # you truly cannot scale any other way (e.g. splitting your app into smaller apps).
-    # Some features, like the API, do not support sharding: they are designed to work
-    # against a single Redis instance only.
+    # thousands of jobs per second.  I don't recommend sharding unless you
+    # cannot scale any other way (e.g. splitting your app into smaller apps).
     def initialize(redis_pool=nil)
       @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
     end
@@ -49,11 +48,13 @@ module Sidekiq
     #   queue - the named queue to use, default 'default'
     #   class - the worker class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
-    #   retry - whether to retry this job if it fails, true or false, default true
+    #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
+    #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
-    # symbols in 'args' will be converted to strings.
+    # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
+    # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
     #
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
@@ -62,19 +63,18 @@ module Sidekiq
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item['class'], normed)
+      payload = process_single(item['class'.freeze], normed)
 
       if payload
         raw_push([payload])
-        payload['jid']
+        payload['jid'.freeze]
       end
     end
 
     ##
     # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing tens of thousands of jobs or more, or if you need
-    # to ensure that a batch doesn't complete prematurely.  This method
-    # basically cuts down on the redis round trip latency.
+    # useful if you are pushing thousands of jobs or more.  This method
+    # cuts out the redis network round trip latency.
     #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
@@ -84,14 +84,19 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
+      arg = items['args'.freeze].first
+      return [] unless arg # no jobs to push
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
+
       normed = normalize_item(items)
-      payloads = items['args'].map do |args|
-        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !args.is_a?(Array)
-        process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f))
+      payloads = items['args'.freeze].map do |args|
+        copy = normed.merge('args'.freeze => args, 'jid'.freeze => SecureRandom.hex(12), 'enqueued_at'.freeze => Time.now.to_f)
+        result = process_single(items['class'.freeze], copy)
+        result ? result : nil
       end.compact
 
       raw_push(payloads) if !payloads.empty?
-      payloads.collect { |payload| payload['jid'] }
+      payloads.collect { |payload| payload['jid'.freeze] }
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -104,13 +109,12 @@ module Sidekiq
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
-    # more than thousands jobs per second.  I do not recommend sharding unless
-    # you truly cannot scale any other way (e.g. splitting your app into smaller apps).
-    # Some features, like the API, do not support sharding: they are designed to work
-    # against a single Redis instance.
+    # thousands of jobs per second.  I do not recommend sharding unless
+    # you cannot scale any other way (e.g. splitting your app into smaller apps).
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
-      raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if x = Thread.current[:sidekiq_via_pool] && x != pool
+      current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
+      raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if current_sidekiq_pool && current_sidekiq_pool != pool
       Thread.current[:sidekiq_via_pool] = pool
       yield
     ensure
@@ -136,14 +140,14 @@ module Sidekiq
       # Messages are enqueued to the 'default' queue.
       #
       def enqueue(klass, *args)
-        klass.client_push('class' => klass, 'args' => args)
+        klass.client_push('class'.freeze => klass, 'args'.freeze => args)
       end
 
       # Example usage:
       #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
-        klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
+        klass.client_push('queue'.freeze => queue, 'class'.freeze => klass, 'args'.freeze => args)
       end
 
       # Example usage:
@@ -154,7 +158,7 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
+        item = { 'class'.freeze => klass, 'args'.freeze => args, 'at'.freeze => ts, 'queue'.freeze => queue }
         item.delete('at'.freeze) if ts <= now
 
         klass.client_push(item)
@@ -180,13 +184,13 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first['at']
+      if payloads.first['at'.freeze]
         conn.zadd('schedule'.freeze, payloads.map do |hash|
           at = hash.delete('at'.freeze).to_s
           [at, Sidekiq.dump_json(hash)]
         end)
       else
-        q = payloads.first['queue']
+        q = payloads.first['queue'.freeze]
         now = Time.now.to_f
         to_push = payloads.map do |entry|
           entry['enqueued_at'.freeze] = now
@@ -198,7 +202,7 @@ module Sidekiq
     end
 
     def process_single(worker_class, item)
-      queue = item['queue']
+      queue = item['queue'.freeze]
 
       middleware.invoke(worker_class, item, queue, @redis_pool) do
         item
@@ -209,6 +213,8 @@ module Sidekiq
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
       raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at'.freeze) && !item['at'].is_a?(Numeric)
+      #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
       normalized_hash(item['class'.freeze])
         .each{ |key, value| item[key] = value if item[key].nil? }

data/lib/sidekiq/core_ext.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 begin
   require 'active_support/core_ext/class/attribute'
 rescue LoadError
@@ -103,3 +104,16 @@ rescue LoadError
 end
 
 
+begin
+  require 'active_support/core_ext/kernel/reporting'
+rescue LoadError
+  module Kernel
+    module_function
+    def silence_warnings
+      old_verbose, $VERBOSE = $VERBOSE, nil
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+  end
+end

data/lib/sidekiq/delay.rb

@@ -0,0 +1,21 @@
+module Sidekiq
+  module Extensions
+
+    def self.enable_delay!
+      if defined?(::ActiveSupport)
+        ActiveSupport.on_load(:active_record) do
+          require 'sidekiq/extensions/active_record'
+          include Sidekiq::Extensions::ActiveRecord
+        end
+        ActiveSupport.on_load(:action_mailer) do
+          require 'sidekiq/extensions/action_mailer'
+          extend Sidekiq::Extensions::ActionMailer
+        end
+      end
+
+      require 'sidekiq/extensions/class_methods'
+      Module.__send__(:include, Sidekiq::Extensions::Klass)
+    end
+
+  end
+end

data/lib/sidekiq/exception_handler.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
@@ -5,7 +6,7 @@ module Sidekiq
 
     class Logger
       def call(ex, ctxHash)
-        Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
+        Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
         Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
         Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
       end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/active_record.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/class_methods.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
 require 'yaml'
 
 module Sidekiq
   module Extensions
+    SIZE_LIMIT = 8_192
+
     class Proxy < BasicObject
       def initialize(performable, target, options={})
         @performable = performable
@@ -16,7 +19,11 @@ module Sidekiq
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
         obj = [@target, name, args]
-        @performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
+        marshalled = ::YAML.dump(obj)
+        if marshalled.size > SIZE_LIMIT
+          ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
+        end
+        @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
       end
     end
 

data/lib/sidekiq/fetch.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
@@ -12,7 +13,7 @@ module Sidekiq
       end
 
       def queue_name
-        queue.gsub(/.*queue:/, ''.freeze)
+        queue.sub(/.*queue:/, ''.freeze)
       end
 
       def requeue

data/lib/sidekiq/job_logger.rb

@@ -0,0 +1,27 @@
+module Sidekiq
+  class JobLogger
+
+    def call(item, queue)
+      begin
+        start = Time.now
+        logger.info("start".freeze)
+        yield
+        logger.info("done: #{elapsed(start)} sec")
+      rescue Exception
+        logger.info("fail: #{elapsed(start)} sec")
+        raise
+      end
+    end
+
+    private
+
+    def elapsed(start)
+      (Time.now - start).round(3)
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+  end
+end
+

data/lib/sidekiq/job_retry.rb

@@ -0,0 +1,235 @@
+require 'sidekiq/scheduled'
+require 'sidekiq/api'
+
+module Sidekiq
+  ##
+  # Automatically retry jobs that fail in Sidekiq.
+  # Sidekiq's retry support assumes a typical development lifecycle:
+  #
+  #   0. Push some code changes with a bug in it.
+  #   1. Bug causes job processing to fail, Sidekiq's middleware captures
+  #      the job and pushes it onto a retry queue.
+  #   2. Sidekiq retries jobs in the retry queue multiple times with
+  #      an exponential delay, the job continues to fail.
+  #   3. After a few days, a developer deploys a fix. The job is
+  #      reprocessed successfully.
+  #   4. Once retries are exhausted, Sidekiq will give up and move the
+  #      job to the Dead Job Queue (aka morgue) where it must be dealt with
+  #      manually in the Web UI.
+  #   5. After 6 months on the DJQ, Sidekiq will discard the job.
+  #
+  # A job looks like:
+  #
+  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+  #
+  # The 'retry' option also accepts a number (in place of 'true'):
+  #
+  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+  #
+  # The job will be retried this number of times before giving up. (If simply
+  # 'true', Sidekiq retries 25 times)
+  #
+  # We'll add a bit more data to the job to support retries:
+  #
+  #  * 'queue' - the queue to use
+  #  * 'retry_count' - number of times we've retried so far.
+  #  * 'error_message' - the message from the exception
+  #  * 'error_class' - the exception class
+  #  * 'failed_at' - the first time it failed
+  #  * 'retried_at' - the last time it was retried
+  #  * 'backtrace' - the number of lines of error backtrace to store
+  #
+  # We don't store the backtrace by default as that can add a lot of overhead
+  # to the job and everyone is using an error service, right?
+  #
+  # The default number of retries is 25 which works out to about 3 weeks
+  # You can change the default maximum number of retries in your initializer:
+  #
+  #   Sidekiq.options[:max_retries] = 7
+  #
+  # or limit the number of retries for a particular worker with:
+  #
+  #    class MyWorker
+  #      include Sidekiq::Worker
+  #      sidekiq_options :retry => 10
+  #    end
+  #
+  class JobRetry
+    class Skip < ::RuntimeError; end
+
+    include Sidekiq::Util
+
+    DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+    def initialize(options = {})
+      @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+    end
+
+    # The global retry handler requires only the barest of data.
+    # We want to be able to retry as much as possible so we don't
+    # require the worker to be instantiated.
+    def global(msg, queue)
+      yield
+    rescue Skip => ex
+      raise ex
+    rescue Sidekiq::Shutdown => ey
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise ey
+    rescue Exception => e
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
+      raise e unless msg['retry']
+      attempt_retry(nil, msg, queue, e)
+      raise e
+    end
+
+
+    # The local retry support means that any errors that occur within
+    # this block can be associated with the given worker instance.
+    # This is required to support the `sidekiq_retries_exhausted` block.
+    #
+    # Note that any exception from the block is wrapped in the Skip
+    # exception so the global block does not reprocess the error.  The
+    # Skip exception is unwrapped within Sidekiq::Processor#process before
+    # calling the handle_exception handlers.
+    def local(worker, msg, queue)
+      yield
+    rescue Skip => ex
+      raise ex
+    rescue Sidekiq::Shutdown => ey
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise ey
+    rescue Exception => e
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
+      if msg['retry'] == nil
+        msg['retry'] = worker.class.get_sidekiq_options['retry']
+      end
+
+      raise e unless msg['retry']
+      attempt_retry(worker, msg, queue, e)
+      # We've handled this error associated with this job, don't
+      # need to handle it at the global level
+      raise Skip
+    end
+
+    private
+
+    # Note that +worker+ can be nil here if an error is raised before we can
+    # instantiate the worker instance.  All access must be guarded and
+    # best effort.
+    def attempt_retry(worker, msg, queue, exception)
+      max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+
+      msg['queue'] = if msg['retry_queue']
+        msg['retry_queue']
+      else
+        queue
+      end
+
+      # App code can stuff all sorts of crazy binary data into the error message
+      # that won't convert to JSON.
+      m = exception.message.to_s[0, 10_000]
+      if m.respond_to?(:scrub!)
+        m.force_encoding("utf-8")
+        m.scrub!
+      end
+
+      msg['error_message'] = m
+      msg['error_class'] = exception.class.name
+      count = if msg['retry_count']
+        msg['retried_at'] = Time.now.to_f
+        msg['retry_count'] += 1
+      else
+        msg['failed_at'] = Time.now.to_f
+        msg['retry_count'] = 0
+      end
+
+      if msg['backtrace'] == true
+        msg['error_backtrace'] = exception.backtrace
+      elsif !msg['backtrace']
+        # do nothing
+      elsif msg['backtrace'].to_i != 0
+        msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+      end
+
+      if count < max_retry_attempts
+        delay = delay_for(worker, count, exception)
+        logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+        retry_at = Time.now.to_f + delay
+        payload = Sidekiq.dump_json(msg)
+        Sidekiq.redis do |conn|
+          conn.zadd('retry', retry_at.to_s, payload)
+        end
+      else
+        # Goodbye dear message, you (re)tried your best I'm sure.
+        retries_exhausted(worker, msg, exception)
+      end
+    end
+
+    def retries_exhausted(worker, msg, exception)
+      logger.debug { "Retries exhausted for job" }
+      begin
+        block = worker && worker.sidekiq_retries_exhausted_block || Sidekiq.default_retries_exhausted
+        block.call(msg, exception) if block
+      rescue => e
+        handle_exception(e, { context: "Error calling retries_exhausted for #{msg['class']}", job: msg })
+      end
+
+      send_to_morgue(msg) unless msg['dead'] == false
+    end
+
+    def send_to_morgue(msg)
+      Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+      payload = Sidekiq.dump_json(msg)
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.zadd('dead', now, payload)
+          conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
+          conn.zremrangebyrank('dead', 0, -DeadSet.max_jobs)
+        end
+      end
+    end
+
+    def retry_attempts_from(msg_retry, default)
+      if msg_retry.is_a?(Integer)
+        msg_retry
+      else
+        default
+      end
+    end
+
+    def delay_for(worker, count, exception)
+      worker && worker.sidekiq_retry_in_block? && retry_in(worker, count, exception) || seconds_to_delay(count)
+    end
+
+    # delayed_job uses the same basic formula
+    def seconds_to_delay(count)
+      (count ** 4) + 15 + (rand(30)*(count+1))
+    end
+
+    def retry_in(worker, count, exception)
+      begin
+        worker.sidekiq_retry_in_block.call(count, exception).to_i
+      rescue Exception => e
+        handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
+        nil
+      end
+    end
+
+    def exception_caused_by_shutdown?(e, checked_causes = [])
+      return false unless e.cause
+
+      # Handle circular causes
+      checked_causes << e.object_id
+      return false if checked_causes.include?(e.cause.object_id)
+
+      e.cause.instance_of?(Sidekiq::Shutdown) ||
+        exception_caused_by_shutdown?(e.cause, checked_causes)
+    end
+
+  end
+end