sidekiq 4.2.10 → 5.0.5

data/lib/sidekiq/cli.rb

@@ -43,6 +43,10 @@ module Sidekiq
       write_pid
     end
 
+    def jruby?
+      defined?(::JRUBY_VERSION)
+    end
+
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
@@ -51,8 +55,14 @@ module Sidekiq
       print_banner
 
       self_read, self_write = IO.pipe
+      sigs = %w(INT TERM TTIN TSTP)
+      # USR1 and USR2 don't work on the JVM
+      if !jruby?
+        sigs << 'USR1'
+        sigs << 'USR2'
+      end
 
-      %w(INT TERM USR1 USR2 TTIN TSTP).each do |sig|
+      sigs.each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -71,6 +81,9 @@ module Sidekiq
       ver = Sidekiq.redis_info['redis_version']
       raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
 
+      # cache process identity
+      Sidekiq.options[:identity] = identity
+
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
       Sidekiq.server_middleware
 
@@ -213,7 +226,6 @@ module Sidekiq
 
       opts[:strict] = true if opts[:strict].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
-      opts[:identity] = identity
 
       options.merge!(opts)
     end
@@ -230,9 +242,7 @@ module Sidekiq
       if File.directory?(options[:require])
         require 'rails'
         if ::Rails::VERSION::MAJOR < 4
-          require 'sidekiq/rails'
-          require File.expand_path("#{options[:require]}/config/environment.rb")
-          ::Rails.application.eager_load!
+          raise "Sidekiq no longer supports this version of Rails"
         elsif ::Rails::VERSION::MAJOR == 4
           # Painful contortions, see 1791 for discussion
           # No autoloading, we want to force eager load for everything.
@@ -243,7 +253,6 @@ module Sidekiq
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
         else
-          # Rails 5+ && development mode, use Reloader
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
         end

data/lib/sidekiq/client.rb

@@ -48,9 +48,15 @@ module Sidekiq
     #   queue - the named queue to use, default 'default'
     #   class - the worker class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
+    #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
+    # If class is set to the class name, the jobs' options will be based on Sidekiq's default
+    # worker options. Otherwise, they will be based on the job class's options.
+    #
+    # Any options valid for a worker class's sidekiq_options are also available here.
+    #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
     # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
@@ -62,11 +68,11 @@ module Sidekiq
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item['class'], normed)
+      payload = process_single(item['class'.freeze], normed)
 
       if payload
         raw_push([payload])
-        payload['jid']
+        payload['jid'.freeze]
       end
     end
 
@@ -83,19 +89,19 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items['args'].first
+      arg = items['args'.freeze].first
       return [] unless arg # no jobs to push
       raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
 
       normed = normalize_item(items)
-      payloads = items['args'].map do |args|
-        copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
-        result = process_single(items['class'], copy)
+      payloads = items['args'.freeze].map do |args|
+        copy = normed.merge('args'.freeze => args, 'jid'.freeze => SecureRandom.hex(12), 'enqueued_at'.freeze => Time.now.to_f)
+        result = process_single(items['class'.freeze], copy)
         result ? result : nil
       end.compact
 
       raw_push(payloads) if !payloads.empty?
-      payloads.collect { |payload| payload['jid'] }
+      payloads.collect { |payload| payload['jid'.freeze] }
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -139,14 +145,14 @@ module Sidekiq
       # Messages are enqueued to the 'default' queue.
       #
       def enqueue(klass, *args)
-        klass.client_push('class' => klass, 'args' => args)
+        klass.client_push('class'.freeze => klass, 'args'.freeze => args)
       end
 
       # Example usage:
       #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
-        klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
+        klass.client_push('queue'.freeze => queue, 'class'.freeze => klass, 'args'.freeze => args)
       end
 
       # Example usage:
@@ -157,7 +163,7 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
+        item = { 'class'.freeze => klass, 'args'.freeze => args, 'at'.freeze => ts, 'queue'.freeze => queue }
         item.delete('at'.freeze) if ts <= now
 
         klass.client_push(item)
@@ -183,13 +189,13 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first['at']
+      if payloads.first['at'.freeze]
         conn.zadd('schedule'.freeze, payloads.map do |hash|
           at = hash.delete('at'.freeze).to_s
           [at, Sidekiq.dump_json(hash)]
         end)
       else
-        q = payloads.first['queue']
+        q = payloads.first['queue'.freeze]
         now = Time.now.to_f
         to_push = payloads.map do |entry|
           entry['enqueued_at'.freeze] = now
@@ -201,7 +207,7 @@ module Sidekiq
     end
 
     def process_single(worker_class, item)
-      queue = item['queue']
+      queue = item['queue'.freeze]
 
       middleware.invoke(worker_class, item, queue, @redis_pool) do
         item
@@ -212,6 +218,7 @@ module Sidekiq
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
       raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at'.freeze) && !item['at'].is_a?(Numeric)
       #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
       normalized_hash(item['class'.freeze])

data/lib/sidekiq/core_ext.rb

@@ -1,119 +1 @@
-# frozen_string_literal: true
-begin
-  require 'active_support/core_ext/class/attribute'
-rescue LoadError
-
-  # A dumbed down version of ActiveSupport's
-  # Class#class_attribute helper.
-  class Class
-    def class_attribute(*attrs)
-      instance_writer = true
-
-      attrs.each do |name|
-        class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def self.#{name}() nil end
-          def self.#{name}?() !!#{name} end
-
-          def self.#{name}=(val)
-            singleton_class.class_eval do
-              define_method(:#{name}) { val }
-            end
-
-            if singleton_class?
-              class_eval do
-                def #{name}
-                  defined?(@#{name}) ? @#{name} : singleton_class.#{name}
-                end
-              end
-            end
-            val
-          end
-
-          def #{name}
-            defined?(@#{name}) ? @#{name} : self.class.#{name}
-          end
-
-          def #{name}?
-            !!#{name}
-          end
-        RUBY
-
-        attr_writer name if instance_writer
-      end
-    end
-
-    private
-    def singleton_class?
-      ancestors.first != self
-    end
-  end
-end
-
-begin
-  require 'active_support/core_ext/hash/keys'
-  require 'active_support/core_ext/hash/deep_merge'
-rescue LoadError
-  class Hash
-    def stringify_keys
-      keys.each do |key|
-        self[key.to_s] = delete(key)
-      end
-      self
-    end if !{}.respond_to?(:stringify_keys)
-
-    def symbolize_keys
-      keys.each do |key|
-        self[(key.to_sym rescue key) || key] = delete(key)
-      end
-      self
-    end if !{}.respond_to?(:symbolize_keys)
-
-    def deep_merge(other_hash, &block)
-      dup.deep_merge!(other_hash, &block)
-    end if !{}.respond_to?(:deep_merge)
-
-    def deep_merge!(other_hash, &block)
-      other_hash.each_pair do |k,v|
-        tv = self[k]
-        if tv.is_a?(Hash) && v.is_a?(Hash)
-          self[k] = tv.deep_merge(v, &block)
-        else
-          self[k] = block && tv ? block.call(k, tv, v) : v
-        end
-      end
-      self
-    end if !{}.respond_to?(:deep_merge!)
-  end
-end
-
-begin
-  require 'active_support/core_ext/string/inflections'
-rescue LoadError
-  class String
-    def constantize
-      names = self.split('::')
-      names.shift if names.empty? || names.first.empty?
-
-      constant = Object
-      names.each do |name|
-        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-      end
-      constant
-    end
-  end if !"".respond_to?(:constantize)
-end
-
-
-begin
-  require 'active_support/core_ext/kernel/reporting'
-rescue LoadError
-  module Kernel
-    module_function
-    def silence_warnings
-      old_verbose, $VERBOSE = $VERBOSE, nil
-      yield
-    ensure
-      $VERBOSE = old_verbose
-    end
-  end
-end
+raise "no longer used, will be removed in 5.1"

data/lib/sidekiq/delay.rb

@@ -0,0 +1,41 @@
+module Sidekiq
+  module Extensions
+
+    def self.enable_delay!
+      if defined?(::ActiveSupport)
+        require 'sidekiq/extensions/active_record'
+        require 'sidekiq/extensions/action_mailer'
+
+        # Need to patch Psych so it can autoload classes whose names are serialized
+        # in the delayed YAML.
+        Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
+
+        ActiveSupport.on_load(:active_record) do
+          include Sidekiq::Extensions::ActiveRecord
+        end
+        ActiveSupport.on_load(:action_mailer) do
+          extend Sidekiq::Extensions::ActionMailer
+        end
+      end
+
+      require 'sidekiq/extensions/class_methods'
+      Module.__send__(:include, Sidekiq::Extensions::Klass)
+    end
+
+    module PsychAutoload
+      def resolve_class(klass_name)
+        return nil if !klass_name || klass_name.empty?
+        # constantize
+        names = klass_name.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      rescue NameError
+        super
+      end
+    end
+  end
+end
+

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -3,6 +3,8 @@ require 'yaml'
 
 module Sidekiq
   module Extensions
+    SIZE_LIMIT = 8_192
+
     class Proxy < BasicObject
       def initialize(performable, target, options={})
         @performable = performable
@@ -17,7 +19,11 @@ module Sidekiq
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
         obj = [@target, name, args]
-        @performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
+        marshalled = ::YAML.dump(obj)
+        if marshalled.size > SIZE_LIMIT
+          ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
+        end
+        @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
       end
     end
 

data/lib/sidekiq/job_logger.rb

@@ -0,0 +1,24 @@
+module Sidekiq
+  class JobLogger
+
+    def call(item, queue)
+      start = Time.now
+      logger.info("start".freeze)
+      yield
+      logger.info("done: #{elapsed(start)} sec")
+    rescue Exception
+      logger.info("fail: #{elapsed(start)} sec")
+      raise
+    end
+
+    private
+
+    def elapsed(start)
+      (Time.now - start).round(3)
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+  end
+end

data/lib/sidekiq/job_retry.rb

@@ -0,0 +1,228 @@
+require 'sidekiq/scheduled'
+require 'sidekiq/api'
+
+module Sidekiq
+  ##
+  # Automatically retry jobs that fail in Sidekiq.
+  # Sidekiq's retry support assumes a typical development lifecycle:
+  #
+  #   0. Push some code changes with a bug in it.
+  #   1. Bug causes job processing to fail, Sidekiq's middleware captures
+  #      the job and pushes it onto a retry queue.
+  #   2. Sidekiq retries jobs in the retry queue multiple times with
+  #      an exponential delay, the job continues to fail.
+  #   3. After a few days, a developer deploys a fix. The job is
+  #      reprocessed successfully.
+  #   4. Once retries are exhausted, Sidekiq will give up and move the
+  #      job to the Dead Job Queue (aka morgue) where it must be dealt with
+  #      manually in the Web UI.
+  #   5. After 6 months on the DJQ, Sidekiq will discard the job.
+  #
+  # A job looks like:
+  #
+  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+  #
+  # The 'retry' option also accepts a number (in place of 'true'):
+  #
+  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+  #
+  # The job will be retried this number of times before giving up. (If simply
+  # 'true', Sidekiq retries 25 times)
+  #
+  # We'll add a bit more data to the job to support retries:
+  #
+  #  * 'queue' - the queue to use
+  #  * 'retry_count' - number of times we've retried so far.
+  #  * 'error_message' - the message from the exception
+  #  * 'error_class' - the exception class
+  #  * 'failed_at' - the first time it failed
+  #  * 'retried_at' - the last time it was retried
+  #  * 'backtrace' - the number of lines of error backtrace to store
+  #
+  # We don't store the backtrace by default as that can add a lot of overhead
+  # to the job and everyone is using an error service, right?
+  #
+  # The default number of retries is 25 which works out to about 3 weeks
+  # You can change the default maximum number of retries in your initializer:
+  #
+  #   Sidekiq.options[:max_retries] = 7
+  #
+  # or limit the number of retries for a particular worker with:
+  #
+  #    class MyWorker
+  #      include Sidekiq::Worker
+  #      sidekiq_options :retry => 10
+  #    end
+  #
+  class JobRetry
+    class Skip < ::RuntimeError; end
+
+    include Sidekiq::Util
+
+    DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+    def initialize(options = {})
+      @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+    end
+
+    # The global retry handler requires only the barest of data.
+    # We want to be able to retry as much as possible so we don't
+    # require the worker to be instantiated.
+    def global(msg, queue)
+      yield
+    rescue Skip => ex
+      raise ex
+    rescue Sidekiq::Shutdown => ey
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise ey
+    rescue Exception => e
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
+      raise e unless msg['retry']
+      attempt_retry(nil, msg, queue, e)
+      raise e
+    end
+
+
+    # The local retry support means that any errors that occur within
+    # this block can be associated with the given worker instance.
+    # This is required to support the `sidekiq_retries_exhausted` block.
+    #
+    # Note that any exception from the block is wrapped in the Skip
+    # exception so the global block does not reprocess the error.  The
+    # Skip exception is unwrapped within Sidekiq::Processor#process before
+    # calling the handle_exception handlers.
+    def local(worker, msg, queue)
+      yield
+    rescue Skip => ex
+      raise ex
+    rescue Sidekiq::Shutdown => ey
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise ey
+    rescue Exception => e
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
+      if msg['retry'] == nil
+        msg['retry'] = worker.class.get_sidekiq_options['retry']
+      end
+
+      raise e unless msg['retry']
+      attempt_retry(worker, msg, queue, e)
+      # We've handled this error associated with this job, don't
+      # need to handle it at the global level
+      raise Skip
+    end
+
+    private
+
+    # Note that +worker+ can be nil here if an error is raised before we can
+    # instantiate the worker instance.  All access must be guarded and
+    # best effort.
+    def attempt_retry(worker, msg, queue, exception)
+      max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+
+      msg['queue'] = if msg['retry_queue']
+        msg['retry_queue']
+      else
+        queue
+      end
+
+      # App code can stuff all sorts of crazy binary data into the error message
+      # that won't convert to JSON.
+      m = exception.message.to_s[0, 10_000]
+      if m.respond_to?(:scrub!)
+        m.force_encoding("utf-8")
+        m.scrub!
+      end
+
+      msg['error_message'] = m
+      msg['error_class'] = exception.class.name
+      count = if msg['retry_count']
+        msg['retried_at'] = Time.now.to_f
+        msg['retry_count'] += 1
+      else
+        msg['failed_at'] = Time.now.to_f
+        msg['retry_count'] = 0
+      end
+
+      if msg['backtrace'] == true
+        msg['error_backtrace'] = exception.backtrace
+      elsif !msg['backtrace']
+        # do nothing
+      elsif msg['backtrace'].to_i != 0
+        msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+      end
+
+      if count < max_retry_attempts
+        delay = delay_for(worker, count, exception)
+        logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+        retry_at = Time.now.to_f + delay
+        payload = Sidekiq.dump_json(msg)
+        Sidekiq.redis do |conn|
+          conn.zadd('retry', retry_at.to_s, payload)
+        end
+      else
+        # Goodbye dear message, you (re)tried your best I'm sure.
+        retries_exhausted(worker, msg, exception)
+      end
+    end
+
+    def retries_exhausted(worker, msg, exception)
+      logger.debug { "Retries exhausted for job" }
+      begin
+        block = worker && worker.sidekiq_retries_exhausted_block || Sidekiq.default_retries_exhausted
+        block.call(msg, exception) if block
+      rescue => e
+        handle_exception(e, { context: "Error calling retries_exhausted for #{msg['class']}", job: msg })
+      end
+
+      send_to_morgue(msg) unless msg['dead'] == false
+    end
+
+    def send_to_morgue(msg)
+      Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+      payload = Sidekiq.dump_json(msg)
+      DeadSet.new.kill(payload)
+    end
+
+    def retry_attempts_from(msg_retry, default)
+      if msg_retry.is_a?(Integer)
+        msg_retry
+      else
+        default
+      end
+    end
+
+    def delay_for(worker, count, exception)
+      worker && worker.sidekiq_retry_in_block && retry_in(worker, count, exception) || seconds_to_delay(count)
+    end
+
+    # delayed_job uses the same basic formula
+    def seconds_to_delay(count)
+      (count ** 4) + 15 + (rand(30)*(count+1))
+    end
+
+    def retry_in(worker, count, exception)
+      begin
+        worker.sidekiq_retry_in_block.call(count, exception).to_i
+      rescue Exception => e
+        handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
+        nil
+      end
+    end
+
+    def exception_caused_by_shutdown?(e, checked_causes = [])
+      return false unless e.cause
+
+      # Handle circular causes
+      checked_causes << e.object_id
+      return false if checked_causes.include?(e.cause.object_id)
+
+      e.cause.instance_of?(Sidekiq::Shutdown) ||
+        exception_caused_by_shutdown?(e.cause, checked_causes)
+    end
+
+  end
+end