sr-sidekiq 4.1.6

data/lib/sidekiq/middleware/server/logging.rb

@@ -0,0 +1,40 @@
+module Sidekiq
+  module Middleware
+    module Server
+      class Logging
+
+        def call(worker, item, queue)
+          Sidekiq::Logging.with_context(log_context(worker, item)) do
+            begin
+              start = Time.now
+              logger.info("start".freeze)
+              yield
+              logger.info("done: #{elapsed(start)} sec")
+            rescue Exception
+              logger.info("fail: #{elapsed(start)} sec")
+              raise
+            end
+          end
+        end
+
+        private
+
+        # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+        # attribute to expose the underlying thing.
+        def log_context(worker, item)
+          klass = item['wrapped'.freeze] || worker.class.to_s
+          "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+        end
+
+        def elapsed(start)
+          (Time.now - start).round(3)
+        end
+
+        def logger
+          Sidekiq.logger
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -0,0 +1,205 @@
+require 'sidekiq/scheduled'
+require 'sidekiq/api'
+
+module Sidekiq
+  module Middleware
+    module Server
+      ##
+      # Automatically retry jobs that fail in Sidekiq.
+      # Sidekiq's retry support assumes a typical development lifecycle:
+      #
+      #   0. Push some code changes with a bug in it.
+      #   1. Bug causes job processing to fail, Sidekiq's middleware captures
+      #      the job and pushes it onto a retry queue.
+      #   2. Sidekiq retries jobs in the retry queue multiple times with
+      #      an exponential delay, the job continues to fail.
+      #   3. After a few days, a developer deploys a fix. The job is
+      #      reprocessed successfully.
+      #   4. Once retries are exhausted, Sidekiq will give up and move the
+      #      job to the Dead Job Queue (aka morgue) where it must be dealt with
+      #      manually in the Web UI.
+      #   5. After 6 months on the DJQ, Sidekiq will discard the job.
+      #
+      # A job looks like:
+      #
+      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+      #
+      # The 'retry' option also accepts a number (in place of 'true'):
+      #
+      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+      #
+      # The job will be retried this number of times before giving up. (If simply
+      # 'true', Sidekiq retries 25 times)
+      #
+      # We'll add a bit more data to the job to support retries:
+      #
+      #  * 'queue' - the queue to use
+      #  * 'retry_count' - number of times we've retried so far.
+      #  * 'error_message' - the message from the exception
+      #  * 'error_class' - the exception class
+      #  * 'failed_at' - the first time it failed
+      #  * 'retried_at' - the last time it was retried
+      #  * 'backtrace' - the number of lines of error backtrace to store
+      #
+      # We don't store the backtrace by default as that can add a lot of overhead
+      # to the job and everyone is using an error service, right?
+      #
+      # The default number of retry attempts is 25 which works out to about 3 weeks
+      # of retries. You can pass a value for the max number of retry attempts when
+      # adding the middleware using the options hash:
+      #
+      #   Sidekiq.configure_server do |config|
+      #     config.server_middleware do |chain|
+      #       chain.add Sidekiq::Middleware::Server::RetryJobs, :max_retries => 7
+      #     end
+      #   end
+      #
+      # or limit the number of retries for a particular worker with:
+      #
+      #    class MyWorker
+      #      include Sidekiq::Worker
+      #      sidekiq_options :retry => 10
+      #    end
+      #
+      class RetryJobs
+        include Sidekiq::Util
+
+        DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+        def initialize(options = {})
+          @max_retries = options.fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+        end
+
+        def call(worker, msg, queue)
+          yield
+        rescue Sidekiq::Shutdown
+          # ignore, will be pushed back onto queue during hard_shutdown
+          raise
+        rescue Exception => e
+          # ignore, will be pushed back onto queue during hard_shutdown
+          raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
+
+          raise e unless msg['retry']
+          attempt_retry(worker, msg, queue, e)
+        end
+
+        private
+
+        def attempt_retry(worker, msg, queue, exception)
+          max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+
+          msg['queue'] = if msg['retry_queue']
+            msg['retry_queue']
+          else
+            queue
+          end
+
+          # App code can stuff all sorts of crazy binary data into the error message
+          # that won't convert to JSON.
+          m = exception.message.to_s[0, 10_000]
+          if m.respond_to?(:scrub!)
+            m.force_encoding("utf-8")
+            m.scrub!
+          end
+
+          msg['error_message'] = m
+          msg['error_class'] = exception.class.name
+          count = if msg['retry_count']
+            msg['retried_at'] = Time.now.to_f
+            msg['retry_count'] += 1
+          else
+            msg['failed_at'] = Time.now.to_f
+            msg['retry_count'] = 0
+          end
+
+          if msg['backtrace'] == true
+            msg['error_backtrace'] = exception.backtrace
+          elsif !msg['backtrace']
+            # do nothing
+          elsif msg['backtrace'].to_i != 0
+            msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+          end
+
+          if count < max_retry_attempts
+            delay = delay_for(worker, count, exception)
+            logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+            retry_at = Time.now.to_f + delay
+            payload = Sidekiq.dump_json(msg)
+            Sidekiq.redis do |conn|
+              conn.zadd('retry', retry_at.to_s, payload)
+            end
+          else
+            # Goodbye dear message, you (re)tried your best I'm sure.
+            retries_exhausted(worker, msg, exception)
+          end
+
+          raise exception
+        end
+
+        def retries_exhausted(worker, msg, exception)
+          logger.debug { "Retries exhausted for job" }
+          begin
+            block = worker.sidekiq_retries_exhausted_block || Sidekiq.default_retries_exhausted
+            block.call(msg, exception) if block
+          rescue => e
+            handle_exception(e, { context: "Error calling retries_exhausted for #{worker.class}", job: msg })
+          end
+
+          send_to_morgue(msg) unless msg['dead'] == false
+        end
+
+        def send_to_morgue(msg)
+          Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+          payload = Sidekiq.dump_json(msg)
+          now = Time.now.to_f
+          Sidekiq.redis do |conn|
+            conn.multi do
+              conn.zadd('dead', now, payload)
+              conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
+              conn.zremrangebyrank('dead', 0, -DeadSet.max_jobs)
+            end
+          end
+        end
+
+        def retry_attempts_from(msg_retry, default)
+          if msg_retry.is_a?(Fixnum)
+            msg_retry
+          else
+            default
+          end
+        end
+
+        def delay_for(worker, count, exception)
+          worker.sidekiq_retry_in_block? && retry_in(worker, count, exception) || seconds_to_delay(count)
+        end
+
+        # delayed_job uses the same basic formula
+        def seconds_to_delay(count)
+          (count ** 4) + 15 + (rand(30)*(count+1))
+        end
+
+        def retry_in(worker, count, exception)
+          begin
+            worker.sidekiq_retry_in_block.call(count, exception).to_i
+          rescue Exception => e
+            handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
+            nil
+          end
+        end
+
+        def exception_caused_by_shutdown?(e, checked_causes = [])
+          # In Ruby 2.1.0 only, check if exception is a result of shutdown.
+          return false unless defined?(e.cause)
+
+          # Handle circular causes
+          checked_causes << e.object_id
+          return false if checked_causes.include?(e.cause.object_id)
+
+          e.cause.instance_of?(Sidekiq::Shutdown) ||
+            exception_caused_by_shutdown?(e.cause, checked_causes)
+        end
+
+      end
+    end
+  end
+end

data/lib/sidekiq/paginator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+module Sidekiq
+  module Paginator
+
+    def page(key, pageidx=1, page_size=25, opts=nil)
+      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      pageidx = current_page - 1
+      total_size = 0
+      items = []
+      starting = pageidx * page_size
+      ending = starting + page_size - 1
+
+      Sidekiq.redis do |conn|
+        type = conn.type(key)
+
+        case type
+        when 'zset'
+          rev = opts && opts[:reverse]
+          total_size, items = conn.multi do
+            conn.zcard(key)
+            if rev
+              conn.zrevrange(key, starting, ending, :with_scores => true)
+            else
+              conn.zrange(key, starting, ending, :with_scores => true)
+            end
+          end
+          [current_page, total_size, items]
+        when 'list'
+          total_size, items = conn.multi do
+            conn.llen(key)
+            conn.lrange(key, starting, ending)
+          end
+          [current_page, total_size, items]
+        when 'none'
+          [1, 0, []]
+        else
+          raise "can't page a #{type}"
+        end
+      end
+    end
+
+  end
+end

data/lib/sidekiq/processor.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+require 'sidekiq/util'
+require 'sidekiq/fetch'
+require 'thread'
+require 'concurrent/map'
+require 'concurrent/atomic/atomic_fixnum'
+
+module Sidekiq
+  ##
+  # The Processor is a standalone thread which:
+  #
+  # 1. fetches a job from Redis
+  # 2. executes the job
+  #   a. instantiate the Worker
+  #   b. run the middleware chain
+  #   c. call #perform
+  #
+  # A Processor can exit due to shutdown (processor_stopped)
+  # or due to an error during job execution (processor_died)
+  #
+  # If an error occurs in the job execution, the
+  # Processor calls the Manager to create a new one
+  # to replace itself and exits.
+  #
+  class Processor
+
+    include Util
+
+    attr_reader :thread
+    attr_reader :job
+
+    def initialize(mgr)
+      @mgr = mgr
+      @down = false
+      @done = false
+      @job = nil
+      @thread = nil
+      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
+    end
+
+    def terminate(wait=false)
+      @done = true
+      return if !@thread
+      @thread.value if wait
+    end
+
+    def kill(wait=false)
+      @done = true
+      return if !@thread
+      # unlike the other actors, terminate does not wait
+      # for the thread to finish because we don't know how
+      # long the job will take to finish.  Instead we
+      # provide a `kill` method to call after the shutdown
+      # timeout passes.
+      @thread.raise ::Sidekiq::Shutdown
+      @thread.value if wait
+    end
+
+    def start
+      @thread ||= safe_thread("processor", &method(:run))
+    end
+
+    private unless $TESTING
+
+    def run
+      begin
+        while !@done
+          process_one
+        end
+        @mgr.processor_stopped(self)
+      rescue Sidekiq::Shutdown
+        @mgr.processor_stopped(self)
+      rescue Exception => ex
+        @mgr.processor_died(self, ex)
+      end
+    end
+
+    def process_one
+      @job = fetch
+      process(@job) if @job
+      @job = nil
+    end
+
+    def get_one
+      begin
+        work = @strategy.retrieve_work
+        (logger.info { "Redis is online, #{Time.now - @down} sec downtime" }; @down = nil) if @down
+        work
+      rescue Sidekiq::Shutdown
+      rescue => ex
+        handle_fetch_exception(ex)
+      end
+    end
+
+    def fetch
+      j = get_one
+      if j && @done
+        j.requeue
+        nil
+      else
+        j
+      end
+    end
+
+    def handle_fetch_exception(ex)
+      if !@down
+        @down = Time.now
+        logger.error("Error fetching job: #{ex}")
+        ex.backtrace.each do |bt|
+          logger.error(bt)
+        end
+      end
+      sleep(1)
+      nil
+    end
+
+    def process(work)
+      jobstr = work.job
+      queue = work.queue_name
+
+      ack = false
+      begin
+        job = Sidekiq.load_json(jobstr)
+        klass  = job['class'.freeze].constantize
+        worker = klass.new
+        worker.jid = job['jid'.freeze]
+
+        stats(worker, job, queue) do
+          Sidekiq.server_middleware.invoke(worker, job, queue) do
+            # Only ack if we either attempted to start this job or
+            # successfully completed it. This prevents us from
+            # losing jobs if a middleware raises an exception before yielding
+            ack = true
+            execute_job(worker, cloned(job['args'.freeze]))
+          end
+        end
+        ack = true
+      rescue Sidekiq::Shutdown
+        # Had to force kill this job because it didn't finish
+        # within the timeout.  Don't acknowledge the work since
+        # we didn't properly finish it.
+        ack = false
+      rescue Exception => ex
+        handle_exception(ex, job || { :job => jobstr })
+        raise
+      ensure
+        work.acknowledge if ack
+      end
+    end
+
+    def execute_job(worker, cloned_args)
+      worker.perform(*cloned_args)
+    end
+
+    def thread_identity
+      @str ||= Thread.current.object_id.to_s(36)
+    end
+
+    WORKER_STATE = Concurrent::Map.new
+    PROCESSED = Concurrent::AtomicFixnum.new
+    FAILURE = Concurrent::AtomicFixnum.new
+
+    def stats(worker, job, queue)
+      tid = thread_identity
+      WORKER_STATE[tid] = {:queue => queue, :payload => job, :run_at => Time.now.to_i }
+
+      begin
+        yield
+      rescue Exception
+        FAILURE.increment
+        raise
+      ensure
+        WORKER_STATE.delete(tid)
+        PROCESSED.increment
+      end
+    end
+
+    # Deep clone the arguments passed to the worker so that if
+    # the job fails, what is pushed back onto Redis hasn't
+    # been mutated by the worker.
+    def cloned(ary)
+      Marshal.load(Marshal.dump(ary))
+    end
+
+  end
+end