sidekiq 3.4.1 → 7.3.0

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -1,206 +0,0 @@
-require 'sidekiq/scheduled'
-require 'sidekiq/api'
-
-module Sidekiq
-  module Middleware
-    module Server
-      ##
-      # Automatically retry jobs that fail in Sidekiq.
-      # Sidekiq's retry support assumes a typical development lifecycle:
-      #
-      #   0. push some code changes with a bug in it
-      #   1. bug causes job processing to fail, sidekiq's middleware captures
-      #      the job and pushes it onto a retry queue
-      #   2. sidekiq retries jobs in the retry queue multiple times with
-      #      an exponential delay, the job continues to fail
-      #   3. after a few days, a developer deploys a fix.  the job is
-      #      reprocessed successfully.
-      #   4. once retries are exhausted, sidekiq will give up and move the
-      #      job to the Dead Job Queue (aka morgue) where it must be dealt with
-      #      manually in the Web UI.
-      #   5. After 6 months on the DJQ, Sidekiq will discard the job.
-      #
-      # A job looks like:
-      #
-      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
-      #
-      # The 'retry' option also accepts a number (in place of 'true'):
-      #
-      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
-      #
-      # The job will be retried this number of times before giving up. (If simply
-      # 'true', Sidekiq retries 25 times)
-      #
-      # We'll add a bit more data to the job to support retries:
-      #
-      #  * 'queue' - the queue to use
-      #  * 'retry_count' - number of times we've retried so far.
-      #  * 'error_message' - the message from the exception
-      #  * 'error_class' - the exception class
-      #  * 'failed_at' - the first time it failed
-      #  * 'retried_at' - the last time it was retried
-      #  * 'backtrace' - the number of lines of error backtrace to store
-      #
-      # We don't store the backtrace by default as that can add a lot of overhead
-      # to the job and everyone is using an error service, right?
-      #
-      # The default number of retry attempts is 25 which works out to about 3 weeks
-      # of retries. You can pass a value for the max number of retry attempts when
-      # adding the middleware using the options hash:
-      #
-      #   Sidekiq.configure_server do |config|
-      #     config.server_middleware do |chain|
-      #       chain.add Sidekiq::Middleware::Server::RetryJobs, :max_retries => 7
-      #     end
-      #   end
-      #
-      # or limit the number of retries for a particular worker with:
-      #
-      #    class MyWorker
-      #      include Sidekiq::Worker
-      #      sidekiq_options :retry => 10
-      #    end
-      #
-      class RetryJobs
-        include Sidekiq::Util
-
-        DEFAULT_MAX_RETRY_ATTEMPTS = 25
-
-        def initialize(options = {})
-          @max_retries = options.fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
-        end
-
-        def call(worker, msg, queue)
-          yield
-        rescue Sidekiq::Shutdown
-          # ignore, will be pushed back onto queue during hard_shutdown
-          raise
-        rescue Exception => e
-          # ignore, will be pushed back onto queue during hard_shutdown
-          raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
-
-          raise e unless msg['retry']
-          attempt_retry(worker, msg, queue, e)
-        end
-
-        private
-
-        def attempt_retry(worker, msg, queue, exception)
-          max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
-
-          msg['queue'] = if msg['retry_queue']
-            msg['retry_queue']
-          else
-            queue
-          end
-
-          # App code can stuff all sorts of crazy binary data into the error message
-          # that won't convert to JSON.
-          m = exception.message[0..10_000]
-          if m.respond_to?(:scrub!)
-            m.force_encoding("utf-8")
-            m.scrub!
-          end
-
-          msg['error_message'] = m
-          msg['error_class'] = exception.class.name
-          count = if msg['retry_count']
-            msg['retried_at'] = Time.now.to_f
-            msg['retry_count'] += 1
-          else
-            msg['failed_at'] = Time.now.to_f
-            msg['retry_count'] = 0
-          end
-
-          if msg['backtrace'] == true
-            msg['error_backtrace'] = exception.backtrace
-          elsif !msg['backtrace']
-            # do nothing
-          elsif msg['backtrace'].to_i != 0
-            msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
-          end
-
-          if count < max_retry_attempts
-            delay = delay_for(worker, count)
-            logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
-            retry_at = Time.now.to_f + delay
-            payload = Sidekiq.dump_json(msg)
-            Sidekiq.redis do |conn|
-              conn.zadd('retry', retry_at.to_s, payload)
-            end
-          else
-            # Goodbye dear message, you (re)tried your best I'm sure.
-            retries_exhausted(worker, msg)
-          end
-
-          raise exception
-        end
-
-        def retries_exhausted(worker, msg)
-          logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
-          begin
-            if worker.sidekiq_retries_exhausted_block?
-              worker.sidekiq_retries_exhausted_block.call(msg)
-            end
-          rescue => e
-            handle_exception(e, { context: "Error calling retries_exhausted for #{worker.class}", job: msg })
-          end
-
-          send_to_morgue(msg) unless msg['dead'] == false
-        end
-
-        def send_to_morgue(msg)
-          Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
-          payload = Sidekiq.dump_json(msg)
-          now = Time.now.to_f
-          Sidekiq.redis do |conn|
-            conn.multi do
-              conn.zadd('dead', now, payload)
-              conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
-              conn.zremrangebyrank('dead', 0, -DeadSet.max_jobs)
-            end
-          end
-        end
-
-        def retry_attempts_from(msg_retry, default)
-          if msg_retry.is_a?(Fixnum)
-            msg_retry
-          else
-            default
-          end
-        end
-
-        def delay_for(worker, count)
-          worker.sidekiq_retry_in_block? && retry_in(worker, count) || seconds_to_delay(count)
-        end
-
-        # delayed_job uses the same basic formula
-        def seconds_to_delay(count)
-          (count ** 4) + 15 + (rand(30)*(count+1))
-        end
-
-        def retry_in(worker, count)
-          begin
-            worker.sidekiq_retry_in_block.call(count)
-          rescue Exception => e
-            handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
-            nil
-          end
-        end
-
-        def exception_caused_by_shutdown?(e, checked_causes = [])
-          # In Ruby 2.1.0 only, check if exception is a result of shutdown.
-          return false unless defined?(e.cause)
-
-          # Handle circular causes
-          checked_causes << e.object_id
-          return false if checked_causes.include?(e.cause.object_id)
-
-          e.cause.instance_of?(Sidekiq::Shutdown) ||
-            exception_caused_by_shutdown?(e.cause, checked_causes)
-        end
-
-      end
-    end
-  end
-end

data/lib/sidekiq/util.rb

@@ -1,55 +0,0 @@
-require 'socket'
-require 'securerandom'
-require 'sidekiq/exception_handler'
-require 'sidekiq/core_ext'
-
-module Sidekiq
-  ##
-  # This module is part of Sidekiq core and not intended for extensions.
-  #
-  module Util
-    include ExceptionHandler
-
-    EXPIRY = 60 * 60 * 24
-
-    def watchdog(last_words)
-      yield
-    rescue Exception => ex
-      handle_exception(ex, { context: last_words })
-      raise ex
-    end
-
-    def logger
-      Sidekiq.logger
-    end
-
-    def redis(&block)
-      Sidekiq.redis(&block)
-    end
-
-    def hostname
-      ENV['DYNO'] || Socket.gethostname
-    end
-
-    def process_nonce
-      @@process_nonce ||= SecureRandom.hex(6)
-    end
-
-    def identity
-      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
-    end
-
-    def fire_event(event, reverse=false)
-      arr = Sidekiq.options[:lifecycle_events][event]
-      arr.reverse! if reverse
-      arr.each do |block|
-        begin
-          block.call
-        rescue => ex
-          handle_exception(ex, { event: event })
-        end
-      end
-    end
-
-  end
-end

data/lib/sidekiq/web_helpers.rb

@@ -1,234 +0,0 @@
-require 'uri'
-
-module Sidekiq
-  # This is not a public API
-  module WebHelpers
-    def strings(lang)
-      @@strings ||= {}
-      @@strings[lang] ||= begin
-        # Allow sidekiq-web extensions to add locale paths
-        # so extensions can be localized
-        settings.locales.each_with_object({}) do |path, global|
-          find_locale_files(lang).each do |file|
-            strs = YAML.load(File.open(file))
-            global.deep_merge!(strs[lang])
-          end
-        end
-      end
-    end
-
-    def locale_files
-      @@locale_files = settings.locales.flat_map do |path|
-        Dir["#{path}/*.yml"]
-      end
-    end
-
-    def find_locale_files(lang)
-      locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
-    end
-
-    # This is a hook for a Sidekiq Pro feature.  Please don't touch.
-    def filtering(*)
-    end
-
-    # This view helper provide ability display you html code in
-    # to head of page. Example:
-    #
-    #   <% add_to_head do %>
-    #     <link rel="stylesheet" .../>
-    #     <meta .../>
-    #   <% end %>
-    #
-    def add_to_head(&block)
-      @head_html ||= []
-      @head_html << block if block_given?
-    end
-
-    def display_custom_head
-      return unless @head_html
-      @head_html.map { |block| capture(&block) }.join
-    end
-
-    # Simple capture method for erb templates. The origin was
-    # capture method from sinatra-contrib library.
-    def capture(&block)
-      block.call
-      eval('', block.binding)
-    end
-
-    # Given a browser request Accept-Language header like
-    # "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2", this function
-    # will return "fr" since that's the first code with a matching
-    # locale in web/locales
-    def locale
-      @locale ||= begin
-        locale = 'en'.freeze
-        languages = request.env['HTTP_ACCEPT_LANGUAGE'.freeze] || 'en'.freeze
-        languages.downcase.split(','.freeze).each do |lang|
-          next if lang == '*'.freeze
-          lang = lang.split(';'.freeze)[0]
-          break locale = lang if find_locale_files(lang).any?
-        end
-        locale
-      end
-    end
-
-    def get_locale
-      strings(locale)
-    end
-
-    def t(msg, options={})
-      string = get_locale[msg] || msg
-      if options.empty?
-        string
-      else
-        string % options
-      end
-    end
-
-    def workers
-      @workers ||= Sidekiq::Workers.new
-    end
-
-    def processes
-      @processes ||= Sidekiq::ProcessSet.new
-    end
-
-    def stats
-      @stats ||= Sidekiq::Stats.new
-    end
-
-    def retries_with_score(score)
-      Sidekiq.redis do |conn|
-        conn.zrangebyscore('retry', score, score)
-      end.map { |msg| Sidekiq.load_json(msg) }
-    end
-
-    def location
-      Sidekiq.redis { |conn| conn.client.location }
-    end
-
-    def redis_connection
-      Sidekiq.redis { |conn| conn.client.id }
-    end
-
-    def namespace
-      @@ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
-    end
-
-    def redis_info
-      Sidekiq.redis do |conn|
-        # admin commands can't go through redis-namespace starting
-        # in redis-namespace 2.0
-        if conn.respond_to?(:namespace)
-          conn.redis.info
-        else
-          conn.info
-        end
-      end
-    end
-
-    def root_path
-      "#{env['SCRIPT_NAME']}/"
-    end
-
-    def current_path
-      @current_path ||= request.path_info.gsub(/^\//,'')
-    end
-
-    def current_status
-      workers.size == 0 ? 'idle' : 'active'
-    end
-
-    def relative_time(time)
-      %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
-    end
-
-    def job_params(job, score)
-      "#{score}-#{job['jid']}"
-    end
-
-    def parse_params(params)
-      score, jid = params.split("-")
-      [score.to_f, jid]
-    end
-
-    SAFE_QPARAMS = %w(page poll)
-
-    # Merge options with current params, filter safe params, and stringify to query string
-    def qparams(options)
-      options = options.stringify_keys
-      params.merge(options).map do |key, value|
-        SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
-      end.join("&")
-    end
-
-    def truncate(text, truncate_after_chars = 2000)
-      truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
-    end
-
-    def display_args(args, truncate_after_chars = 2000)
-      args.map do |arg|
-        a = arg.inspect
-        h(truncate(a))
-      end.join(", ")
-    end
-
-    RETRY_JOB_KEYS = Set.new(%w(
-      queue class args retry_count retried_at failed_at
-      jid error_message error_class backtrace
-      error_backtrace enqueued_at retry wrapped
-    ))
-
-    def retry_extra_items(retry_job)
-      @retry_extra_items ||= {}.tap do |extra|
-        retry_job.item.each do |key, value|
-          extra[key] = value unless RETRY_JOB_KEYS.include?(key)
-        end
-      end
-    end
-
-    def number_with_delimiter(number)
-      begin
-        Float(number)
-      rescue ArgumentError, TypeError
-        return number
-      end
-
-      options = {delimiter: ',', separator: '.'}
-      parts = number.to_s.to_str.split('.')
-      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-      parts.join(options[:separator])
-    end
-
-    def h(text)
-      ::Rack::Utils.escape_html(text)
-    rescue ArgumentError => e
-      raise unless e.message.eql?('invalid byte sequence in UTF-8')
-      text.encode!('UTF-16', 'UTF-8', invalid: :replace, replace: '').encode!('UTF-8', 'UTF-16')
-      retry
-    end
-
-    # Any paginated list that performs an action needs to redirect
-    # back to the proper page after performing that action.
-    def redirect_with_query(url)
-      r = request.referer
-      if r && r =~ /\?/
-        ref = URI(r)
-        redirect("#{url}?#{ref.query}")
-      else
-        redirect url
-      end
-    end
-
-    def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-
-      "[#{environment.upcase}] " unless environment == "production"
-    end
-
-    def product_version
-      "Sidekiq v#{Sidekiq::VERSION}"
-    end
-  end
-end

data/lib/sidekiq/worker.rb

@@ -1,89 +0,0 @@
-require 'sidekiq/client'
-require 'sidekiq/core_ext'
-
-module Sidekiq
-
-  ##
-  # Include this module in your worker class and you can easily create
-  # asynchronous jobs:
-  #
-  # class HardWorker
-  #   include Sidekiq::Worker
-  #
-  #   def perform(*args)
-  #     # do some work
-  #   end
-  # end
-  #
-  # Then in your Rails app, you can do this:
-  #
-  #   HardWorker.perform_async(1, 2, 3)
-  #
-  # Note that perform_async is a class method, perform is an instance method.
-  module Worker
-    attr_accessor :jid
-
-    def self.included(base)
-      base.extend(ClassMethods)
-      base.class_attribute :sidekiq_options_hash
-      base.class_attribute :sidekiq_retry_in_block
-      base.class_attribute :sidekiq_retries_exhausted_block
-    end
-
-    def logger
-      Sidekiq.logger
-    end
-
-    module ClassMethods
-
-      def perform_async(*args)
-        client_push('class' => self, 'args' => args)
-      end
-
-      def perform_in(interval, *args)
-        int = interval.to_f
-        now = Time.now
-        ts = (int < 1_000_000_000 ? (now + interval).to_f : int)
-
-        item = { 'class' => self, 'args' => args, 'at' => ts }
-
-        # Optimization to enqueue something now that is scheduled to go out now or in the past
-        item.delete('at'.freeze) if ts <= now.to_f
-
-        client_push(item)
-      end
-      alias_method :perform_at, :perform_in
-
-      ##
-      # Allows customization for this type of Worker.
-      # Legal options:
-      #
-      #   :queue - use a named queue for this Worker, default 'default'
-      #   :retry - enable the RetryJobs middleware for this Worker, default *true*
-      #   :backtrace - whether to save any error backtrace in the retry payload to display in web UI,
-      #      can be true, false or an integer number of lines to save, default *false*
-      #   :pool - use the given Redis connection pool to push this type of job to a given shard.
-      def sidekiq_options(opts={})
-        self.sidekiq_options_hash = get_sidekiq_options.merge(opts.stringify_keys)
-      end
-
-      def sidekiq_retry_in(&block)
-        self.sidekiq_retry_in_block = block
-      end
-
-      def sidekiq_retries_exhausted(&block)
-        self.sidekiq_retries_exhausted_block = block
-      end
-
-      def get_sidekiq_options # :nodoc:
-        self.sidekiq_options_hash ||= Sidekiq.default_worker_options
-      end
-
-      def client_push(item) # :nodoc:
-        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
-        Sidekiq::Client.new(pool).push(item.stringify_keys)
-      end
-
-    end
-  end
-end

data/test/config.yml

@@ -1,9 +0,0 @@
----
-:verbose:      false
-:require:      ./test/fake_env.rb
-:pidfile:      /tmp/sidekiq-config-test.pid
-:logfile:      /tmp/sidekiq.log
-:concurrency:  50
-:queues:
-  - [<%="very_"%>often, 2]
-  - [seldom, 1]

data/test/env_based_config.yml

@@ -1,11 +0,0 @@
----
-:pidfile:      /tmp/sidekiq-config-test.pid
-:concurrency:  50
-staging:
-  :verbose:      false
-  :require:      ./test/fake_env.rb
-  :logfile:      /tmp/sidekiq.log
-  :concurrency:  5
-  :queues:
-    - [<%="very_"%>often, 2]
-    - [seldom, 1]

data/test/fixtures/en.yml

@@ -1,2 +0,0 @@
-en:
-  translated_text: 'Changed text from add locals'

data/test/helper.rb

@@ -1,39 +0,0 @@
-$TESTING = true
-require 'coveralls'
-Coveralls.wear! do
-  add_filter "/test/"
-  add_filter "/myapp/"
-end
-
-ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
-if ENV.has_key?("SIMPLECOV")
-  require 'simplecov'
-  SimpleCov.start do
-    add_filter "/test/"
-    add_filter "/myapp/"
-  end
-end
-
-begin
-  require 'pry-byebug'
-rescue LoadError
-end
-
-require 'minitest/autorun'
-require 'minitest/pride'
-
-require 'celluloid/test'
-Celluloid.boot
-require 'sidekiq'
-require 'sidekiq/util'
-Sidekiq.logger.level = Logger::ERROR
-
-Sidekiq::Test = Minitest::Test
-
-require 'sidekiq/redis_connection'
-REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost/15'
-REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'testy')
-
-Sidekiq.configure_client do |config|
-  config.redis = { :url => REDIS_URL, :namespace => 'testy' }
-end