sidekiq 2.15.1 → 4.2.10

data/lib/sidekiq/middleware/chain.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack
@@ -49,13 +50,16 @@ module Sidekiq
   #   end
   # end
   #
-  # This is an example of a minimal client middleware:
+  # This is an example of a minimal client middleware, note
+  # the method must return the result or the job will not push
+  # to Redis:
   #
   # class MyClientHook
-  #   def call(worker_class, msg, queue)
+  #   def call(worker_class, msg, queue, redis_pool)
   #     puts "Before push"
-  #     yield
+  #     result = yield
   #     puts "After push"
+  #     result
   #   end
   # end
   #
@@ -64,6 +68,10 @@ module Sidekiq
       include Enumerable
       attr_reader :entries
 
+      def initialize_copy(copy)
+        copy.instance_variable_set(:@entries, entries.dup)
+      end
+
       def each(&block)
         entries.each(&block)
       end
@@ -82,6 +90,11 @@ module Sidekiq
         entries << Entry.new(klass, *args)
       end
 
+      def prepend(klass, *args)
+        remove(klass) if exists?(klass)
+        entries.insert(0, Entry.new(klass, *args))
+      end
+
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
@@ -108,11 +121,11 @@ module Sidekiq
         entries.clear
       end
 
-      def invoke(*args, &final_action)
+      def invoke(*args)
         chain = retrieve.dup
         traverse_chain = lambda do
           if chain.empty?
-            final_action.call
+            yield
           else
             chain.shift.call(*args, &traverse_chain)
           end

data/lib/sidekiq/middleware/i18n.rb

@@ -1,8 +1,15 @@
+# frozen_string_literal: true
+#
+# Simple middleware to save the current locale and restore it when the job executes.
+# Use it by requiring it in your initializer:
+#
+#     require 'sidekiq/middleware/i18n'
+#
 module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.
   class Client
-    def call(worker_class, msg, queue)
+    def call(worker_class, msg, queue, redis_pool)
       msg['locale'] ||= I18n.locale
       yield
     end
@@ -14,7 +21,7 @@ module Sidekiq::Middleware::I18n
       I18n.locale = msg['locale'] || I18n.default_locale
       yield
     ensure
-      I18n.locale = nil
+      I18n.locale = I18n.default_locale
     end
   end
 end

data/lib/sidekiq/middleware/server/active_record.rb

@@ -5,7 +5,7 @@ module Sidekiq
         def call(*args)
           yield
         ensure
-          ::ActiveRecord::Base.clear_active_connections! if defined?(::ActiveRecord)
+          ::ActiveRecord::Base.clear_active_connections!
         end
       end
     end

data/lib/sidekiq/middleware/server/logging.rb

@@ -4,21 +4,21 @@ module Sidekiq
       class Logging
 
         def call(worker, item, queue)
-          Sidekiq::Logging.with_context("#{worker.class.to_s} JID-#{item['jid']}") do
-            begin
-              start = Time.now
-              logger.info { "start" }
-              yield
-              logger.info { "done: #{elapsed(start)} sec" }
-            rescue Exception
-              logger.info { "fail: #{elapsed(start)} sec" }
-              raise
-            end
+          begin
+            start = Time.now
+            logger.info("start".freeze)
+            yield
+            logger.info("done: #{elapsed(start)} sec")
+          rescue Exception
+            logger.info("fail: #{elapsed(start)} sec")
+            raise
           end
         end
 
+        private
+
         def elapsed(start)
-          (Time.now - start).to_f.round(3)
+          (Time.now - start).round(3)
         end
 
         def logger

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -1,4 +1,5 @@
 require 'sidekiq/scheduled'
+require 'sidekiq/api'
 
 module Sidekiq
   module Middleware
@@ -6,21 +7,22 @@ module Sidekiq
       ##
       # Automatically retry jobs that fail in Sidekiq.
       # Sidekiq's retry support assumes a typical development lifecycle:
-      # 0. push some code changes with a bug in it
-      # 1. bug causes message processing to fail, sidekiq's middleware captures
-      #    the message and pushes it onto a retry queue
-      # 2. sidekiq retries messages in the retry queue multiple times with
-      #    an exponential delay, the message continues to fail
-      # 3. after a few days, a developer deploys a fix.  the message is
-      #    reprocessed successfully.
-      # 4. if 3 never happens, sidekiq will eventually give up and throw the
-      #    message away. If the worker defines a method called 'retries_exhausted',
-      #    this will be called before throwing the message away. If the
-      #    'retries_exhausted' method throws an exception, it's dropped and logged.
       #
-      # A message looks like:
+      #   0. Push some code changes with a bug in it.
+      #   1. Bug causes job processing to fail, Sidekiq's middleware captures
+      #      the job and pushes it onto a retry queue.
+      #   2. Sidekiq retries jobs in the retry queue multiple times with
+      #      an exponential delay, the job continues to fail.
+      #   3. After a few days, a developer deploys a fix. The job is
+      #      reprocessed successfully.
+      #   4. Once retries are exhausted, Sidekiq will give up and move the
+      #      job to the Dead Job Queue (aka morgue) where it must be dealt with
+      #      manually in the Web UI.
+      #   5. After 6 months on the DJQ, Sidekiq will discard the job.
       #
-      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'] }
+      # A job looks like:
+      #
+      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
       #
       # The 'retry' option also accepts a number (in place of 'true'):
       #
@@ -29,7 +31,7 @@ module Sidekiq
       # The job will be retried this number of times before giving up. (If simply
       # 'true', Sidekiq retries 25 times)
       #
-      # We'll add a bit more data to the message to support retries:
+      # We'll add a bit more data to the job to support retries:
       #
       #  * 'queue' - the queue to use
       #  * 'retry_count' - number of times we've retried so far.
@@ -37,18 +39,28 @@ module Sidekiq
       #  * 'error_class' - the exception class
       #  * 'failed_at' - the first time it failed
       #  * 'retried_at' - the last time it was retried
+      #  * 'backtrace' - the number of lines of error backtrace to store
       #
-      # We don't store the backtrace as that can add a lot of overhead
-      # to the message and everyone is using Airbrake, right?
+      # We don't store the backtrace by default as that can add a lot of overhead
+      # to the job and everyone is using an error service, right?
       #
-      # The default number of retry attempts is 25. You can pass a value for the
-      # number of retry attempts when adding the middleware using the options hash:
+      # The default number of retry attempts is 25 which works out to about 3 weeks
+      # of retries. You can pass a value for the max number of retry attempts when
+      # adding the middleware using the options hash:
       #
       #   Sidekiq.configure_server do |config|
       #     config.server_middleware do |chain|
-      #       chain.add Middleware::Server::RetryJobs, {:max_retries => 7}
+      #       chain.add Sidekiq::Middleware::Server::RetryJobs, :max_retries => 7
       #     end
       #   end
+      #
+      # or limit the number of retries for a particular worker with:
+      #
+      #    class MyWorker
+      #      include Sidekiq::Worker
+      #      sidekiq_options :retry => 10
+      #    end
+      #
       class RetryJobs
         include Sidekiq::Util
 
@@ -64,7 +76,16 @@ module Sidekiq
           # ignore, will be pushed back onto queue during hard_shutdown
           raise
         rescue Exception => e
+          # ignore, will be pushed back onto queue during hard_shutdown
+          raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
           raise e unless msg['retry']
+          attempt_retry(worker, msg, queue, e)
+        end
+
+        private
+
+        def attempt_retry(worker, msg, queue, exception)
           max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
 
           msg['queue'] = if msg['retry_queue']
@@ -72,26 +93,35 @@ module Sidekiq
           else
             queue
           end
-          msg['error_message'] = e.message
-          msg['error_class'] = e.class.name
+
+          # App code can stuff all sorts of crazy binary data into the error message
+          # that won't convert to JSON.
+          m = exception.message.to_s[0, 10_000]
+          if m.respond_to?(:scrub!)
+            m.force_encoding("utf-8")
+            m.scrub!
+          end
+
+          msg['error_message'] = m
+          msg['error_class'] = exception.class.name
           count = if msg['retry_count']
-            msg['retried_at'] = Time.now.utc
+            msg['retried_at'] = Time.now.to_f
             msg['retry_count'] += 1
           else
-            msg['failed_at'] = Time.now.utc
+            msg['failed_at'] = Time.now.to_f
             msg['retry_count'] = 0
           end
 
           if msg['backtrace'] == true
-            msg['error_backtrace'] = e.backtrace
-          elsif msg['backtrace'] == false
+            msg['error_backtrace'] = exception.backtrace
+          elsif !msg['backtrace']
             # do nothing
           elsif msg['backtrace'].to_i != 0
-            msg['error_backtrace'] = e.backtrace[0..msg['backtrace'].to_i]
+            msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
           end
 
           if count < max_retry_attempts
-            delay = delay_for(worker, count)
+            delay = delay_for(worker, count, exception)
             logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
             retry_at = Time.now.to_f + delay
             payload = Sidekiq.dump_json(msg)
@@ -100,35 +130,47 @@ module Sidekiq
             end
           else
             # Goodbye dear message, you (re)tried your best I'm sure.
-            retries_exhausted(worker, msg)
+            retries_exhausted(worker, msg, exception)
           end
 
-          raise e
+          raise exception
         end
 
-        def retries_exhausted(worker, msg)
-          logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
-          if worker.respond_to?(:retries_exhausted)
-            logger.warn { "Defining #{worker.class.name}#retries_exhausted as a method is deprecated, use `sidekiq_retries_exhausted` callback instead http://git.io/Ijju8g" }
-            worker.retries_exhausted(*msg['args'])
-          elsif worker.sidekiq_retries_exhausted_block?
-            worker.sidekiq_retries_exhausted_block.call(msg)
+        def retries_exhausted(worker, msg, exception)
+          logger.debug { "Retries exhausted for job" }
+          begin
+            block = worker.sidekiq_retries_exhausted_block || Sidekiq.default_retries_exhausted
+            block.call(msg, exception) if block
+          rescue => e
+            handle_exception(e, { context: "Error calling retries_exhausted for #{worker.class}", job: msg })
           end
 
-        rescue Exception => e
-          handle_exception(e, { :context => "Error calling retries_exhausted" })
+          send_to_morgue(msg) unless msg['dead'] == false
+        end
+
+        def send_to_morgue(msg)
+          Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+          payload = Sidekiq.dump_json(msg)
+          now = Time.now.to_f
+          Sidekiq.redis do |conn|
+            conn.multi do
+              conn.zadd('dead', now, payload)
+              conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
+              conn.zremrangebyrank('dead', 0, -DeadSet.max_jobs)
+            end
+          end
         end
 
         def retry_attempts_from(msg_retry, default)
-          if msg_retry.is_a?(Fixnum)
+          if msg_retry.is_a?(Integer)
             msg_retry
           else
             default
           end
         end
 
-        def delay_for(worker, count)
-          worker.sidekiq_retry_in_block? && retry_in(worker, count) || seconds_to_delay(count)
+        def delay_for(worker, count, exception)
+          worker.sidekiq_retry_in_block? && retry_in(worker, count, exception) || seconds_to_delay(count)
         end
 
         # delayed_job uses the same basic formula
@@ -136,15 +178,27 @@ module Sidekiq
           (count ** 4) + 15 + (rand(30)*(count+1))
         end
 
-        def retry_in(worker, count)
+        def retry_in(worker, count, exception)
           begin
-            worker.sidekiq_retry_in_block.call(count)
+            worker.sidekiq_retry_in_block.call(count, exception).to_i
           rescue Exception => e
-            logger.error { "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default: #{e.message}"}
+            handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
             nil
           end
         end
 
+        def exception_caused_by_shutdown?(e, checked_causes = [])
+          # In Ruby 2.1.0 only, check if exception is a result of shutdown.
+          return false unless defined?(e.cause)
+
+          # Handle circular causes
+          checked_causes << e.object_id
+          return false if checked_causes.include?(e.cause.object_id)
+
+          e.cause.instance_of?(Sidekiq::Shutdown) ||
+            exception_caused_by_shutdown?(e.cause, checked_causes)
+        end
+
       end
     end
   end

data/lib/sidekiq/paginator.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
 module Sidekiq
   module Paginator
-    def page(key, pageidx=1, page_size=25)
+
+    def page(key, pageidx=1, page_size=25, opts=nil)
       current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
       pageidx = current_page - 1
       total_size = 0
@@ -13,19 +15,29 @@ module Sidekiq
 
         case type
         when 'zset'
-          total_size = conn.zcard(key)
-          items = conn.zrange(key, starting, ending, :with_scores => true)
+          rev = opts && opts[:reverse]
+          total_size, items = conn.multi do
+            conn.zcard(key)
+            if rev
+              conn.zrevrange(key, starting, ending, :with_scores => true)
+            else
+              conn.zrange(key, starting, ending, :with_scores => true)
+            end
+          end
+          [current_page, total_size, items]
         when 'list'
-          total_size = conn.llen(key)
-          items = conn.lrange(key, starting, ending)
+          total_size, items = conn.multi do
+            conn.llen(key)
+            conn.lrange(key, starting, ending)
+          end
+          [current_page, total_size, items]
         when 'none'
-          return [1, 0, []]
+          [1, 0, []]
         else
           raise "can't page a #{type}"
         end
       end
-
-      [current_page, total_size, items]
     end
+
   end
 end

data/lib/sidekiq/processor.rb

@@ -1,140 +1,201 @@
+# frozen_string_literal: true
 require 'sidekiq/util'
-require 'sidekiq/actor'
-
-require 'sidekiq/middleware/server/active_record'
-require 'sidekiq/middleware/server/retry_jobs'
-require 'sidekiq/middleware/server/logging'
+require 'sidekiq/fetch'
+require 'thread'
+require 'concurrent/map'
+require 'concurrent/atomic/atomic_fixnum'
 
 module Sidekiq
   ##
-  # The Processor receives a message from the Manager and actually
-  # processes it.  It instantiates the worker, runs the middleware
-  # chain and then calls Sidekiq::Worker#perform.
+  # The Processor is a standalone thread which:
+  #
+  # 1. fetches a job from Redis
+  # 2. executes the job
+  #   a. instantiate the Worker
+  #   b. run the middleware chain
+  #   c. call #perform
+  #
+  # A Processor can exit due to shutdown (processor_stopped)
+  # or due to an error during job execution (processor_died)
+  #
+  # If an error occurs in the job execution, the
+  # Processor calls the Manager to create a new one
+  # to replace itself and exits.
+  #
   class Processor
-    STATS_TIMEOUT = 180 * 24 * 60 * 60
 
     include Util
-    include Actor
 
-    def self.default_middleware
-      Middleware::Chain.new do |m|
-        m.add Middleware::Server::Logging
-        m.add Middleware::Server::RetryJobs
-        m.add Middleware::Server::ActiveRecord
-      end
+    attr_reader :thread
+    attr_reader :job
+
+    def initialize(mgr)
+      @mgr = mgr
+      @down = false
+      @done = false
+      @job = nil
+      @thread = nil
+      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
+      @reloader = Sidekiq.options[:reloader]
+      @executor = Sidekiq.options[:executor]
     end
 
-    attr_accessor :proxy_id
-
-    def initialize(boss)
-      @boss = boss
+    def terminate(wait=false)
+      @done = true
+      return if !@thread
+      @thread.value if wait
     end
 
-    def process(work)
-      msgstr = work.message
-      queue = work.queue_name
+    def kill(wait=false)
+      @done = true
+      return if !@thread
+      # unlike the other actors, terminate does not wait
+      # for the thread to finish because we don't know how
+      # long the job will take to finish.  Instead we
+      # provide a `kill` method to call after the shutdown
+      # timeout passes.
+      @thread.raise ::Sidekiq::Shutdown
+      @thread.value if wait
+    end
 
-      do_defer do
-        @boss.async.real_thread(proxy_id, Thread.current)
+    def start
+      @thread ||= safe_thread("processor", &method(:run))
+    end
 
-        begin
-          msg = Sidekiq.load_json(msgstr)
-          klass  = msg['class'].constantize
-          worker = klass.new
-          worker.jid = msg['jid']
+    private unless $TESTING
 
-          stats(worker, msg, queue) do
-            Sidekiq.server_middleware.invoke(worker, msg, queue) do
-              worker.perform(*cloned(msg['args']))
-            end
-          end
-        rescue Sidekiq::Shutdown
-          # Had to force kill this job because it didn't finish
-          # within the timeout.
-        rescue Exception => ex
-          handle_exception(ex, msg || { :message => msgstr })
-          raise
-        ensure
-          work.acknowledge
+    def run
+      begin
+        while !@done
+          process_one
         end
+        @mgr.processor_stopped(self)
+      rescue Sidekiq::Shutdown
+        @mgr.processor_stopped(self)
+      rescue Exception => ex
+        @mgr.processor_died(self, ex)
       end
-
-      @boss.async.processor_done(current_actor)
     end
 
-    def inspect
-      "<Processor##{object_id.to_s(16)}>"
+    def process_one
+      @job = fetch
+      process(@job) if @job
+      @job = nil
     end
 
-    private
-
-    # We use Celluloid's defer to workaround tiny little
-    # Fiber stacks (4kb!) in MRI 1.9.
-    #
-    # For some reason, Celluloid's thread dispatch, TaskThread,
-    # is unstable under heavy concurrency but TaskFiber has proven
-    # itself stable.
-    NEED_DEFER = (RUBY_ENGINE == 'ruby' && RUBY_VERSION < '2.0.0')
+    def get_one
+      begin
+        work = @strategy.retrieve_work
+        (logger.info { "Redis is online, #{Time.now - @down} sec downtime" }; @down = nil) if @down
+        work
+      rescue Sidekiq::Shutdown
+      rescue => ex
+        handle_fetch_exception(ex)
+      end
+    end
 
-    def do_defer(&block)
-      if NEED_DEFER
-        defer(&block)
+    def fetch
+      j = get_one
+      if j && @done
+        j.requeue
+        nil
       else
-        yield
+        j
       end
     end
 
-    def identity
-      @str ||= "#{hostname}:#{process_id}-#{Thread.current.object_id}:default"
+    def handle_fetch_exception(ex)
+      if !@down
+        @down = Time.now
+        logger.error("Error fetching job: #{ex}")
+        ex.backtrace.each do |bt|
+          logger.error(bt)
+        end
+      end
+      sleep(1)
+      nil
     end
 
-    def stats(worker, msg, queue)
-      redis do |conn|
-        conn.multi do
-          conn.sadd('workers', identity)
-          conn.setex("worker:#{identity}:started", EXPIRY, Time.now.to_s)
-          hash = {:queue => queue, :payload => msg, :run_at => Time.now.to_i }
-          conn.setex("worker:#{identity}", EXPIRY, Sidekiq.dump_json(hash))
+    def process(work)
+      jobstr = work.job
+      queue = work.queue_name
+
+      ack = false
+      begin
+        job_hash = Sidekiq.load_json(jobstr)
+        @reloader.call do
+          klass  = job_hash['class'.freeze].constantize
+          worker = klass.new
+          worker.jid = job_hash['jid'.freeze]
+
+          stats(worker, job_hash, queue) do
+            Sidekiq::Logging.with_context(log_context(job_hash)) do
+              ack = true
+              Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
+                @executor.call do
+                  # Only ack if we either attempted to start this job or
+                  # successfully completed it. This prevents us from
+                  # losing jobs if a middleware raises an exception before yielding
+                  execute_job(worker, cloned(job_hash['args'.freeze]))
+                end
+              end
+            end
+          end
+          ack = true
         end
+      rescue Sidekiq::Shutdown
+        # Had to force kill this job because it didn't finish
+        # within the timeout.  Don't acknowledge the work since
+        # we didn't properly finish it.
+        ack = false
+      rescue Exception => ex
+        handle_exception(ex, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
+        raise
+      ensure
+        work.acknowledge if ack
       end
+    end
+
+    # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+    # attribute to expose the underlying thing.
+    def log_context(item)
+      klass = item['wrapped'.freeze] || item['class'.freeze]
+      "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+    end
+
+    def execute_job(worker, cloned_args)
+      worker.perform(*cloned_args)
+    end
+
+    def thread_identity
+      @str ||= Thread.current.object_id.to_s(36)
+    end
+
+    WORKER_STATE = Concurrent::Map.new
+    PROCESSED = Concurrent::AtomicFixnum.new
+    FAILURE = Concurrent::AtomicFixnum.new
+
+    def stats(worker, job_hash, queue)
+      tid = thread_identity
+      WORKER_STATE[tid] = {:queue => queue, :payload => cloned(job_hash), :run_at => Time.now.to_i }
 
       begin
         yield
       rescue Exception
-        redis do |conn|
-          failed = "stat:failed:#{Time.now.utc.to_date}"
-          result = conn.multi do
-            conn.incrby("stat:failed", 1)
-            conn.incrby(failed, 1)
-          end
-          conn.expire(failed, STATS_TIMEOUT) if result.last == 1
-        end
+        FAILURE.increment
         raise
       ensure
-        redis do |conn|
-          processed = "stat:processed:#{Time.now.utc.to_date}"
-          result = conn.multi do
-            conn.srem("workers", identity)
-            conn.del("worker:#{identity}")
-            conn.del("worker:#{identity}:started")
-            conn.incrby("stat:processed", 1)
-            conn.incrby(processed, 1)
-          end
-          conn.expire(processed, STATS_TIMEOUT) if result.last == 1
-        end
+        WORKER_STATE.delete(tid)
+        PROCESSED.increment
       end
     end
 
-    # Singleton classes are not clonable.
-    SINGLETON_CLASSES = [ NilClass, TrueClass, FalseClass, Symbol, Fixnum, Float, Bignum ].freeze
-
-    # Clone the arguments passed to the worker so that if
-    # the message fails, what is pushed back onto Redis hasn't
+    # Deep clone the arguments passed to the worker so that if
+    # the job fails, what is pushed back onto Redis hasn't
     # been mutated by the worker.
     def cloned(ary)
-      ary.map do |val|
-        SINGLETON_CLASSES.include?(val.class) ? val : val.clone
-      end
+      Marshal.load(Marshal.dump(ary))
     end
+
   end
 end