sidekiq 4.2.10 → 5.0.0.beta1

data/lib/sidekiq/cli.rb

@@ -43,6 +43,10 @@ module Sidekiq
       write_pid
     end
 
+    def jruby?
+      defined?(::JRUBY_VERSION)
+    end
+
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
@@ -51,8 +55,14 @@ module Sidekiq
       print_banner
 
       self_read, self_write = IO.pipe
+      sigs = %w(INT TERM TTIN TSTP)
+      # USR1 and USR2 don't work on the JVM
+      if !jruby?
+        sigs << 'USR1'
+        sigs << 'USR2'
+      end
 
-      %w(INT TERM USR1 USR2 TTIN TSTP).each do |sig|
+      sigs.each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -230,9 +240,7 @@ module Sidekiq
       if File.directory?(options[:require])
         require 'rails'
         if ::Rails::VERSION::MAJOR < 4
-          require 'sidekiq/rails'
-          require File.expand_path("#{options[:require]}/config/environment.rb")
-          ::Rails.application.eager_load!
+          raise "Sidekiq no longer supports this version of Rails"
         elsif ::Rails::VERSION::MAJOR == 4
           # Painful contortions, see 1791 for discussion
           # No autoloading, we want to force eager load for everything.
@@ -243,7 +251,6 @@ module Sidekiq
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
         else
-          # Rails 5+ && development mode, use Reloader
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
         end

data/lib/sidekiq/delay.rb

@@ -0,0 +1,21 @@
+module Sidekiq
+  module Extensions
+
+    def self.enable_delay!
+      if defined?(::ActiveSupport)
+        ActiveSupport.on_load(:active_record) do
+          require 'sidekiq/extensions/active_record'
+          include Sidekiq::Extensions::ActiveRecord
+        end
+        ActiveSupport.on_load(:action_mailer) do
+          require 'sidekiq/extensions/action_mailer'
+          extend Sidekiq::Extensions::ActionMailer
+        end
+      end
+
+      require 'sidekiq/extensions/class_methods'
+      Module.__send__(:include, Sidekiq::Extensions::Klass)
+    end
+
+  end
+end

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -3,6 +3,8 @@ require 'yaml'
 
 module Sidekiq
   module Extensions
+    SIZE_LIMIT = 8_192
+
     class Proxy < BasicObject
       def initialize(performable, target, options={})
         @performable = performable
@@ -17,7 +19,11 @@ module Sidekiq
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
         obj = [@target, name, args]
-        @performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
+        marshalled = ::YAML.dump(obj)
+        if marshalled.size > SIZE_LIMIT
+          ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
+        end
+        @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
       end
     end
 

data/lib/sidekiq/job_logger.rb

@@ -0,0 +1,36 @@
+module Sidekiq
+  class JobLogger
+
+    def call(item, queue)
+      Sidekiq::Logging.with_context(log_context(item)) do
+        begin
+          start = Time.now
+          logger.info("start".freeze)
+          yield
+          logger.info("done: #{elapsed(start)} sec")
+        rescue Exception
+          logger.info("fail: #{elapsed(start)} sec")
+          raise
+        end
+      end
+    end
+
+    private
+
+    # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+    # attribute to expose the underlying thing.
+    def log_context(item)
+      klass = item['wrapped'.freeze] || item["class".freeze]
+      "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+    end
+
+    def elapsed(start)
+      (Time.now - start).round(3)
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+  end
+end
+

data/lib/sidekiq/job_retry.rb

@@ -0,0 +1,232 @@
+require 'sidekiq/scheduled'
+require 'sidekiq/api'
+
+module Sidekiq
+  ##
+  # Automatically retry jobs that fail in Sidekiq.
+  # Sidekiq's retry support assumes a typical development lifecycle:
+  #
+  #   0. Push some code changes with a bug in it.
+  #   1. Bug causes job processing to fail, Sidekiq's middleware captures
+  #      the job and pushes it onto a retry queue.
+  #   2. Sidekiq retries jobs in the retry queue multiple times with
+  #      an exponential delay, the job continues to fail.
+  #   3. After a few days, a developer deploys a fix. The job is
+  #      reprocessed successfully.
+  #   4. Once retries are exhausted, Sidekiq will give up and move the
+  #      job to the Dead Job Queue (aka morgue) where it must be dealt with
+  #      manually in the Web UI.
+  #   5. After 6 months on the DJQ, Sidekiq will discard the job.
+  #
+  # A job looks like:
+  #
+  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+  #
+  # The 'retry' option also accepts a number (in place of 'true'):
+  #
+  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+  #
+  # The job will be retried this number of times before giving up. (If simply
+  # 'true', Sidekiq retries 25 times)
+  #
+  # We'll add a bit more data to the job to support retries:
+  #
+  #  * 'queue' - the queue to use
+  #  * 'retry_count' - number of times we've retried so far.
+  #  * 'error_message' - the message from the exception
+  #  * 'error_class' - the exception class
+  #  * 'failed_at' - the first time it failed
+  #  * 'retried_at' - the last time it was retried
+  #  * 'backtrace' - the number of lines of error backtrace to store
+  #
+  # We don't store the backtrace by default as that can add a lot of overhead
+  # to the job and everyone is using an error service, right?
+  #
+  # The default number of retries is 25 which works out to about 3 weeks
+  # You can change the default maximum number of retries in your initializer:
+  #
+  #   Sidekiq.options[:max_retries] = 7
+  #
+  # or limit the number of retries for a particular worker with:
+  #
+  #    class MyWorker
+  #      include Sidekiq::Worker
+  #      sidekiq_options :retry => 10
+  #    end
+  #
+  class JobRetry
+    class Skip < ::RuntimeError; end
+
+    include Sidekiq::Util
+
+    DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+    def initialize(options = {})
+      @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+    end
+
+    # The global retry handler requires only the barest of data.
+    # We want to be able to retry as much as possible so we don't
+    # require the worker to be instantiated.
+    def global(msg, queue)
+      yield
+    rescue Skip
+      raise
+    rescue Sidekiq::Shutdown
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise
+    rescue Exception => e
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
+      raise e unless msg['retry']
+      attempt_retry(nil, msg, queue, e)
+    end
+
+
+    # The local retry support means that any errors that occur within
+    # this block can be associated with the given worker instance.
+    # This is required to support the `sidekiq_retries_exhausted` block.
+    def local(worker, msg, queue)
+      yield
+    rescue Skip
+      raise
+    rescue Sidekiq::Shutdown
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise
+    rescue Exception => e
+      # ignore, will be pushed back onto queue during hard_shutdown
+      raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
+      if msg['retry'] == nil
+        msg['retry'] = worker.class.get_sidekiq_options['retry']
+      end
+
+      raise e unless msg['retry']
+      attempt_retry(worker, msg, queue, e)
+      # We've handled this error associated with this job, don't
+      # need to handle it at the global level
+      raise Skip
+    end
+
+    private
+
+    # Note that +worker+ can be nil here if an error is raised before we can
+    # instantiate the worker instance.  All access must be guarded and
+    # best effort.
+    def attempt_retry(worker, msg, queue, exception)
+      max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+
+      msg['queue'] = if msg['retry_queue']
+        msg['retry_queue']
+      else
+        queue
+      end
+
+      # App code can stuff all sorts of crazy binary data into the error message
+      # that won't convert to JSON.
+      m = exception.message.to_s[0, 10_000]
+      if m.respond_to?(:scrub!)
+        m.force_encoding("utf-8")
+        m.scrub!
+      end
+
+      msg['error_message'] = m
+      msg['error_class'] = exception.class.name
+      count = if msg['retry_count']
+        msg['retried_at'] = Time.now.to_f
+        msg['retry_count'] += 1
+      else
+        msg['failed_at'] = Time.now.to_f
+        msg['retry_count'] = 0
+      end
+
+      if msg['backtrace'] == true
+        msg['error_backtrace'] = exception.backtrace
+      elsif !msg['backtrace']
+        # do nothing
+      elsif msg['backtrace'].to_i != 0
+        msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+      end
+
+      if count < max_retry_attempts
+        delay = delay_for(worker, count, exception)
+        logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+        retry_at = Time.now.to_f + delay
+        payload = Sidekiq.dump_json(msg)
+        Sidekiq.redis do |conn|
+          conn.zadd('retry', retry_at.to_s, payload)
+        end
+      else
+        # Goodbye dear message, you (re)tried your best I'm sure.
+        retries_exhausted(worker, msg, exception)
+      end
+
+      raise exception
+    end
+
+    def retries_exhausted(worker, msg, exception)
+      logger.debug { "Retries exhausted for job" }
+      begin
+        block = worker && worker.sidekiq_retries_exhausted_block || Sidekiq.default_retries_exhausted
+        block.call(msg, exception) if block
+      rescue => e
+        handle_exception(e, { context: "Error calling retries_exhausted for #{msg['class']}", job: msg })
+      end
+
+      send_to_morgue(msg) unless msg['dead'] == false
+    end
+
+    def send_to_morgue(msg)
+      Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+      payload = Sidekiq.dump_json(msg)
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.zadd('dead', now, payload)
+          conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
+          conn.zremrangebyrank('dead', 0, -DeadSet.max_jobs)
+        end
+      end
+    end
+
+    def retry_attempts_from(msg_retry, default)
+      if msg_retry.is_a?(Integer)
+        msg_retry
+      else
+        default
+      end
+    end
+
+    def delay_for(worker, count, exception)
+      worker && worker.sidekiq_retry_in_block? && retry_in(worker, count, exception) || seconds_to_delay(count)
+    end
+
+    # delayed_job uses the same basic formula
+    def seconds_to_delay(count)
+      (count ** 4) + 15 + (rand(30)*(count+1))
+    end
+
+    def retry_in(worker, count, exception)
+      begin
+        worker.sidekiq_retry_in_block.call(count, exception).to_i
+      rescue Exception => e
+        handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
+        nil
+      end
+    end
+
+    def exception_caused_by_shutdown?(e, checked_causes = [])
+      # In Ruby 2.1.0 only, check if exception is a result of shutdown.
+      return false unless defined?(e.cause)
+
+      # Handle circular causes
+      checked_causes << e.object_id
+      return false if checked_causes.include?(e.cause.object_id)
+
+      e.cause.instance_of?(Sidekiq::Shutdown) ||
+        exception_caused_by_shutdown?(e.cause, checked_causes)
+    end
+
+  end
+end

data/lib/sidekiq/launcher.rb

@@ -61,8 +61,6 @@ module Sidekiq
 
     private unless $TESTING
 
-    JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
-
     def heartbeat
       results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, to_data) }
       results.compact!
@@ -110,11 +108,7 @@ module Sidekiq
 
         return unless msg
 
-        if JVM_RESERVED_SIGNALS.include?(msg)
-          Sidekiq::CLI.instance.handle_signal(msg)
-        else
-          ::Process.kill(msg, $$)
-        end
+        ::Process.kill(msg, $$)
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")

data/lib/sidekiq/middleware/server/active_record.rb

@@ -2,6 +2,15 @@ module Sidekiq
   module Middleware
     module Server
       class ActiveRecord
+
+        def initialize
+          # With Rails 5+ we must use the Reloader **always**.
+          # The reloader handles code loading and db connection management.
+          if ::Rails::VERSION::MAJOR >= 5
+            raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
+          end
+        end
+
         def call(*args)
           yield
         ensure

data/lib/sidekiq/processor.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 require 'sidekiq/util'
 require 'sidekiq/fetch'
+require 'sidekiq/job_logger'
+require 'sidekiq/job_retry'
 require 'thread'
 require 'concurrent/map'
 require 'concurrent/atomic/atomic_fixnum'
@@ -37,7 +39,8 @@ module Sidekiq
       @thread = nil
       @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
       @reloader = Sidekiq.options[:reloader]
-      @executor = Sidekiq.options[:executor]
+      @logging = Sidekiq::JobLogger.new
+      @retrier = Sidekiq::JobRetry.new
     end
 
     def terminate(wait=false)
@@ -116,32 +119,59 @@ module Sidekiq
       nil
     end
 
+    def dispatch(job_hash, queue)
+      # since middleware can mutate the job hash
+      # we clone here so we report the original
+      # job structure to the Web UI
+      pristine = cloned(job_hash)
+
+      @retrier.global(job_hash, queue) do
+        @logging.call(job_hash, queue) do
+          stats(pristine, queue) do
+            # Rails 5 requires a Reloader to wrap code execution.  In order to
+            # constantize the worker and instantiate an instance, we have to call
+            # the Reloader.  It handles code loading, db connection management, etc.
+            # Effectively this block denotes a "unit of work" to Rails.
+            @reloader.call do
+              klass  = job_hash['class'.freeze].constantize
+              worker = klass.new
+              worker.jid = job_hash['jid'.freeze]
+              @retrier.local(worker, job_hash, queue) do
+                yield worker
+              end
+            end
+          end
+        end
+      end
+    end
+
     def process(work)
       jobstr = work.job
       queue = work.queue_name
 
       ack = false
       begin
-        job_hash = Sidekiq.load_json(jobstr)
-        @reloader.call do
-          klass  = job_hash['class'.freeze].constantize
-          worker = klass.new
-          worker.jid = job_hash['jid'.freeze]
-
-          stats(worker, job_hash, queue) do
-            Sidekiq::Logging.with_context(log_context(job_hash)) do
-              ack = true
-              Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
-                @executor.call do
-                  # Only ack if we either attempted to start this job or
-                  # successfully completed it. This prevents us from
-                  # losing jobs if a middleware raises an exception before yielding
-                  execute_job(worker, cloned(job_hash['args'.freeze]))
-                end
-              end
-            end
-          end
+        # Treat malformed JSON like a process crash -- don't acknowledge it.
+        # * In Sidekiq, the error will be logged but job discarded.
+        # * In Sidekiq Pro, the error will be logged and the job retried when
+        #   it is recovered by the reliability algorithm.  The job may act like
+        #   a poison pill and never execute until manually removed but job loss
+        #   is considered worse.
+        job_hash = nil
+        begin
+          job_hash = Sidekiq.load_json(jobstr)
+        rescue => ex
+          Sidekiq.logger.error { "Pushing job to dead queue due to invalid JSON: #{ex}" }
+          send_to_morgue(jobstr)
           ack = true
+          raise
+        end
+
+        ack = true
+        dispatch(job_hash, queue) do |worker|
+          Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
+            execute_job(worker, cloned(job_hash['args'.freeze]))
+          end
         end
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
@@ -156,11 +186,15 @@ module Sidekiq
       end
     end
 
-    # If we're using a wrapper class, like ActiveJob, use the "wrapped"
-    # attribute to expose the underlying thing.
-    def log_context(item)
-      klass = item['wrapped'.freeze] || item['class'.freeze]
-      "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+    def send_to_morgue(msg)
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.zadd('dead', now, msg)
+          conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
+          conn.zremrangebyrank('dead', 0, -DeadSet.max_jobs)
+        end
+      end
     end
 
     def execute_job(worker, cloned_args)
@@ -175,9 +209,9 @@ module Sidekiq
     PROCESSED = Concurrent::AtomicFixnum.new
     FAILURE = Concurrent::AtomicFixnum.new
 
-    def stats(worker, job_hash, queue)
+    def stats(job_hash, queue)
       tid = thread_identity
-      WORKER_STATE[tid] = {:queue => queue, :payload => cloned(job_hash), :run_at => Time.now.to_i }
+      WORKER_STATE[tid] = {:queue => queue, :payload => job_hash, :run_at => Time.now.to_i }
 
       begin
         yield
@@ -193,8 +227,8 @@ module Sidekiq
     # Deep clone the arguments passed to the worker so that if
     # the job fails, what is pushed back onto Redis hasn't
     # been mutated by the worker.
-    def cloned(ary)
-      Marshal.load(Marshal.dump(ary))
+    def cloned(thing)
+      Marshal.load(Marshal.dump(thing))
     end
 
   end