sidekiq 4.2.4 → 5.2.0

data/lib/sidekiq/rails.rb

@@ -1,36 +1,5 @@
 # frozen_string_literal: true
 module Sidekiq
-  def self.hook_rails!
-    return if defined?(@delay_removed)
-
-    ActiveSupport.on_load(:active_record) do
-      include Sidekiq::Extensions::ActiveRecord
-    end
-
-    ActiveSupport.on_load(:action_mailer) do
-      extend Sidekiq::Extensions::ActionMailer
-    end
-
-    Module.__send__(:include, Sidekiq::Extensions::Klass)
-  end
-
-  # Removes the generic aliases which MAY clash with names of already
-  #  created methods by other applications. The methods `sidekiq_delay`,
-  #  `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
-  def self.remove_delay!
-    @delay_removed = true
-
-    [Extensions::ActiveRecord,
-     Extensions::ActionMailer,
-     Extensions::Klass].each do |mod|
-      mod.module_eval do
-        remove_method :delay if respond_to?(:delay)
-        remove_method :delay_for if respond_to?(:delay_for)
-        remove_method :delay_until if respond_to?(:delay_until)
-      end
-    end
-  end
-
   class Rails < ::Rails::Engine
     # We need to setup this up before any application configuration which might
     # change Sidekiq middleware.
@@ -48,26 +17,16 @@ module Sidekiq
       end
     end
 
-    initializer 'sidekiq' do
-      Sidekiq.hook_rails!
-    end
-
-    # We have to add the reloader after initialize to see if cache_classes has
-    # been turned on.
-    #
-    # This hook happens after all initialziers are run, just before returning
-    # from config/environment.rb back to sidekiq/cli.rb.
     config.after_initialize do
-      if ::Rails::VERSION::MAJOR >= 5
-        # The reloader also takes care of ActiveRecord but is incompatible with
-        # the ActiveRecord middleware so make sure it's not in the chain already.
-        if defined?(Sidekiq::Middleware::Server::ActiveRecord) && Sidekiq.server_middleware.exists?(Sidekiq::Middleware::Server::ActiveRecord)
-          raise ArgumentError, "You are using the Sidekiq ActiveRecord middleware and the new Rails 5 reloader which are incompatible. Please remove the ActiveRecord middleware from your Sidekiq middleware configuration."
-        elsif ::Rails.application.config.cache_classes
-          # The reloader API has proven to be troublesome under load in production.
-          # We won't use it at all when classes are cached, see #3154
-          Sidekiq.logger.debug { "Autoload disabled in #{::Rails.env}, Sidekiq will not reload changed classes" }
-        else
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      # We have to add the reloader after initialize to see if cache_classes has
+      # been turned on.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      #
+      Sidekiq.configure_server do |_|
+        if ::Rails::VERSION::MAJOR >= 5
           Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
         end
       end
@@ -75,7 +34,6 @@ module Sidekiq
 
     class Reloader
       def initialize(app = ::Rails.application)
-        Sidekiq.logger.debug "Enabling Rails 5+ live code reloading, so hot!" unless app.config.cache_classes
         @app = app
       end
 
@@ -91,3 +49,9 @@ module Sidekiq
     end
   end if defined?(::Rails)
 end
+
+if defined?(::Rails) && ::Rails::VERSION::MAJOR < 4
+  $stderr.puts("**************************************************")
+  $stderr.puts("⛔️ WARNING: Sidekiq server is no longer supported by Rails 3.2 - please ensure your server/workers are updated")
+  $stderr.puts("**************************************************")
+end

data/lib/sidekiq/redis_connection.rb

@@ -8,11 +8,22 @@ module Sidekiq
     class << self
 
       def create(options={})
-        options = options.symbolize_keys
+        options.keys.each do |key|
+          options[key.to_sym] = options.delete(key)
+        end
 
+        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{$$}" if !options.has_key?(:id)
         options[:url] ||= determine_redis_provider
 
-        size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
+        size = if options[:size]
+                 options[:size]
+               elsif Sidekiq.server?
+                 Sidekiq.options[:concurrency] + 5
+               elsif ENV['RAILS_MAX_THREADS']
+                 Integer(ENV['RAILS_MAX_THREADS'])
+               else
+                 5
+               end
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
@@ -35,7 +46,7 @@ module Sidekiq
       #   - enterprise's leader election
       #   - enterprise's cron support
       def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size <= concurrency
       end
 
       def build_client(options)
@@ -67,7 +78,14 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] = opts[:driver] || 'ruby'
+        opts[:driver] ||= 'ruby'
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
 
         opts
       end
@@ -91,7 +109,15 @@ module Sidekiq
       end
 
       def determine_redis_provider
-        ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+        # If you have this in your environment:
+        # MY_REDIS_URL=redis://hostname.example.com:1238/4
+        # then set:
+        # REDIS_PROVIDER=MY_REDIS_URL
+        # and Sidekiq will find your custom URL variable with no custom
+        # initialization code at all.
+        ENV[
+          ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+        ]
       end
 
     end

data/lib/sidekiq/scheduled.rb

@@ -17,7 +17,7 @@ module Sidekiq
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
-            while job = conn.zrangebyscore(sorted_set, '-inf'.freeze, now, :limit => [0, 1]).first do
+            while job = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
 
               # Pop item off the queue and add it to the work queue. If the job can't be popped from
               # the queue, it's because another process already popped it so we can move on to the
@@ -79,9 +79,7 @@ module Sidekiq
           # Most likely a problem with redis networking.
           # Punt and try again at the next interval
           logger.error ex.message
-          ex.backtrace.each do |bt|
-            logger.error(bt)
-          end
+          handle_exception(ex)
         end
       end
 
@@ -95,13 +93,38 @@ module Sidekiq
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
         logger.error ex.message
-        logger.error ex.backtrace.first
+        handle_exception(ex)
         sleep 5
       end
 
-      # Calculates a random interval that is ±50% the desired average.
       def random_poll_interval
-        poll_interval_average * rand + poll_interval_average.to_f / 2
+        # We want one Sidekiq process to schedule jobs every N seconds.  We have M processes
+        # and **don't** want to coordinate.
+        #
+        # So in N*M second timespan, we want each process to schedule once.  The basic loop is:
+        #
+        # * sleep a random amount within that N*M timespan
+        # * wake up and schedule
+        #
+        # We want to avoid one edge case: imagine a set of 2 processes, scheduling every 5 seconds,
+        # so N*M = 10.  Each process decides to randomly sleep 8 seconds, now we've failed to meet
+        # that 5 second average. Thankfully each schedule cycle will sleep randomly so the next
+        # iteration could see each process sleep for 1 second, undercutting our average.
+        #
+        # So below 10 processes, we special case and ensure the processes sleep closer to the average.
+        # In the example above, each process should schedule every 10 seconds on average. We special
+        # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
+        # As we run more processes, the scheduling interval average will approach an even spread
+        # between 0 and poll interval so we don't need this artifical boost.
+        #
+        if process_count < 10
+          # For small clusters, calculate a random interval that is ±50% the desired average.
+          poll_interval_average * rand + poll_interval_average.to_f / 2
+        else
+          # With 10+ processes, we should have enough randomness to get decent polling
+          # across the entire timespan
+          poll_interval_average * rand
+        end
       end
 
       # We do our best to tune the poll interval to the size of the active Sidekiq
@@ -125,9 +148,13 @@ module Sidekiq
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
       def scaled_poll_interval
+        process_count * Sidekiq.options[:average_scheduled_poll_interval]
+      end
+
+      def process_count
         pcount = Sidekiq::ProcessSet.new.size
         pcount = 1 if pcount == 0
-        pcount * Sidekiq.options[:average_scheduled_poll_interval]
+        pcount
       end
 
       def initial_wait

data/lib/sidekiq/testing.rb

@@ -55,6 +55,15 @@ module Sidekiq
         yield @server_chain if block_given?
         @server_chain
       end
+
+      def constantize(str)
+        names = str.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      end
     end
   end
 
@@ -63,29 +72,31 @@ module Sidekiq
 
   class EmptyQueueError < RuntimeError; end
 
-  class Client
-    alias_method :raw_push_real, :raw_push
-
+  module TestingClient
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          Queues.push(job['queue'], job['class'], Sidekiq.load_json(Sidekiq.dump_json(job)))
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
+          Queues.push(job['queue'], job['class'], job)
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = job['class'].constantize
+          klass = Sidekiq::Testing.constantize(job['class'])
           job['id'] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
         end
         true
       else
-        raw_push_real(payloads)
+        super
       end
     end
   end
 
+  Sidekiq::Client.prepend TestingClient
+
   module Queues
     ##
     # The Queues class is only for testing the fake queue implementation.
@@ -307,10 +318,16 @@ module Sidekiq
           worker_classes = jobs.map { |job| job["class"] }.uniq
 
           worker_classes.each do |worker_class|
-            worker_class.constantize.drain
+            Sidekiq::Testing.constantize(worker_class).drain
           end
         end
       end
     end
   end
 end
+
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
+  puts("**************************************************")
+  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
+  puts("**************************************************")
+end

data/lib/sidekiq/util.rb

@@ -2,7 +2,6 @@
 require 'socket'
 require 'securerandom'
 require 'sidekiq/exception_handler'
-require 'sidekiq/core_ext'
 
 module Sidekiq
   ##
@@ -22,6 +21,7 @@ module Sidekiq
 
     def safe_thread(name, &block)
       Thread.new do
+        Thread.current['sidekiq_label'] = name
         watchdog(name, &block)
       end
     end
@@ -46,7 +46,10 @@ module Sidekiq
       @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
-    def fire_event(event, reverse=false)
+    def fire_event(event, options={})
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+
       arr = Sidekiq.options[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
@@ -54,6 +57,7 @@ module Sidekiq
           block.call
         rescue => ex
           handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
+          raise ex if reraise
         end
       end
       arr.clear

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "4.2.4"
+  VERSION = "5.2.0"
 end

data/lib/sidekiq/web/action.rb

@@ -2,7 +2,7 @@
 
 module Sidekiq
   class WebAction
-    RACK_SESSION = 'rack.session'.freeze
+    RACK_SESSION = 'rack.session'
 
     attr_accessor :env, :block, :type
 
@@ -39,10 +39,6 @@ module Sidekiq
       env[RACK_SESSION]
     end
 
-    def content_type(type)
-      @type = type
-    end
-
     def erb(content, options = {})
       if content.kind_of? Symbol
         unless respond_to?(:"_erb_#{content}")
@@ -81,7 +77,7 @@ module Sidekiq
     private
 
     def _erb(file, locals)
-      locals.each {|k, v| define_singleton_method(k){ v } } if locals
+      locals.each {|k, v| define_singleton_method(k){ v } unless (singleton_methods.include? k)} if locals
 
       if file.kind_of?(String)
         ERB.new(file).result(binding)

data/lib/sidekiq/web/application.rb

@@ -4,9 +4,24 @@ module Sidekiq
   class WebApplication
     extend WebRouter
 
-    CONTENT_LENGTH = "Content-Length".freeze
-    CONTENT_TYPE = "Content-Type".freeze
+    CONTENT_LENGTH = "Content-Length"
+    CONTENT_TYPE = "Content-Type"
     REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+    CSP_HEADER = [
+      "default-src 'self' https: http:",
+      "child-src 'self'",
+      "connect-src 'self' https: http: wss: ws:",
+      "font-src 'self' https: http:",
+      "frame-src 'self'",
+      "img-src 'self' https: http: data:",
+      "manifest-src 'self'",
+      "media-src 'self'",
+      "object-src 'none'",
+      "script-src 'self' https: http:",
+      "style-src 'self' https: http: 'unsafe-inline'",
+      "worker-src 'self'",
+      "base-uri 'self'"
+    ].join('; ').freeze
 
     def initialize(klass)
       @klass = klass
@@ -28,10 +43,6 @@ module Sidekiq
       # nothing, backwards compatibility
     end
 
-    get "" do
-      redirect(root_path)
-    end
-
     get "/" do
       @redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
       stats_history = Sidekiq::Stats::History.new((params['days'] || 30).to_i)
@@ -89,7 +100,7 @@ module Sidekiq
       name = route_params[:name]
       Sidekiq::Job.new(params['key_val'], name).delete
 
-      redirect_with_query("#{root_path}queues/#{name}")
+      redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
     end
 
     get '/morgue' do
@@ -238,7 +249,6 @@ module Sidekiq
     get '/stats' do
       sidekiq_stats = Sidekiq::Stats.new
       redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
-
       json(
         sidekiq: {
           processed:       sidekiq_stats.processed,
@@ -251,7 +261,8 @@ module Sidekiq
           dead:            sidekiq_stats.dead_size,
           default_latency: sidekiq_stats.default_queue_latency
         },
-        redis: redis_stats
+        redis: redis_stats,
+        server_utc_time: server_utc_time
       )
     end
 
@@ -278,19 +289,15 @@ module Sidekiq
       resp = case resp
       when Array
         resp
-      when Fixnum
-        [resp, {}, []]
       else
-        type_header = case action.type
-        when :json
-          { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }
-        when String
-          { "Content-Type" => action.type, "Cache-Control" => "no-cache" }
-        else
-          { "Content-Type" => "text/html", "Cache-Control" => "no-cache" }
-        end
-
-        [200, type_header, [resp]]
+        headers = {
+          "Content-Type" => "text/html",
+          "Cache-Control" => "no-cache",
+          "Content-Language" => action.locale,
+          "Content-Security-Policy" => CSP_HEADER
+        }
+
+        [200, headers, [resp]]
       end
 
       resp[1] = resp[1].dup

data/lib/sidekiq/web/helpers.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 require 'uri'
+require 'set'
 require 'yaml'
+require 'cgi'
 
 module Sidekiq
   # This is not a public API
@@ -13,7 +15,7 @@ module Sidekiq
         settings.locales.each_with_object({}) do |path, global|
           find_locale_files(lang).each do |file|
             strs = YAML.load(File.open(file))
-            global.deep_merge!(strs[lang])
+            global.merge!(strs[lang])
           end
         end
       end
@@ -22,6 +24,7 @@ module Sidekiq
     def clear_caches
       @@strings = nil
       @@locale_files = nil
+      @@available_locales = nil
     end
 
     def locale_files
@@ -30,6 +33,10 @@ module Sidekiq
       end
     end
 
+    def available_locales
+      @@available_locales ||= locale_files.map { |path| File.basename(path, '.yml') }.uniq
+    end
+
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
@@ -63,23 +70,52 @@ module Sidekiq
       end
     end
 
-    # Given a browser request Accept-Language header like
-    # "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2", this function
-    # will return "fr" since that's the first code with a matching
-    # locale in web/locales
+    def text_direction
+      get_locale['TextDirection'] || 'ltr'
+    end
+
+    def rtl?
+      text_direction == 'rtl'
+    end
+
+    # See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+    def user_preferred_languages
+      languages = env['HTTP_ACCEPT_LANGUAGE']
+      languages.to_s.downcase.gsub(/\s+/, '').split(',').map do |language|
+        locale, quality = language.split(';q=', 2)
+        locale  = nil if locale == '*' # Ignore wildcards
+        quality = quality ? quality.to_f : 1.0
+        [locale, quality]
+      end.sort do |(_, left), (_, right)|
+        right <=> left
+      end.map(&:first).compact
+    end
+
+    # Given an Accept-Language header like "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2"
+    # this method will try to best match the available locales to the user's preferred languages.
+    #
+    # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
       @locale ||= begin
-        locale = 'en'.freeze
-        languages = env['HTTP_ACCEPT_LANGUAGE'.freeze] || 'en'.freeze
-        languages.downcase.split(','.freeze).each do |lang|
-          next if lang == '*'.freeze
-          lang = lang.split(';'.freeze)[0]
-          break locale = lang if find_locale_files(lang).any?
-        end
-        locale
+        matched_locale = user_preferred_languages.map do |preferred|
+          preferred_language = preferred.split('-', 2).first
+
+          lang_group = available_locales.select do |available|
+            preferred_language == available.split('-', 2).first
+          end
+
+          lang_group.find { |lang| lang == preferred } || lang_group.min_by(&:length)
+        end.compact.first
+
+        matched_locale || 'en'
       end
     end
 
+    # mperham/sidekiq#3243
+    def unfiltered?
+      yield unless env['PATH_INFO'].start_with?("/filter/")
+    end
+
     def get_locale
       strings(locale)
     end
@@ -111,16 +147,15 @@ module Sidekiq
       end.map { |msg| Sidekiq.load_json(msg) }
     end
 
-    def location
-      Sidekiq.redis { |conn| conn.client.location }
-    end
-
     def redis_connection
-      Sidekiq.redis { |conn| conn.client.id }
+      Sidekiq.redis do |conn|
+        c = conn.connection
+        "redis://#{c[:location]}/#{c[:db]}"
+      end
     end
 
     def namespace
-      @@ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
+      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
     end
 
     def redis_info
@@ -140,7 +175,8 @@ module Sidekiq
     end
 
     def relative_time(time)
-      %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
+      stamp = time.getutc.iso8601
+      %{<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>}
     end
 
     def job_params(job, score)
@@ -148,7 +184,7 @@ module Sidekiq
     end
 
     def parse_params(params)
-      score, jid = params.split("-")
+      score, jid = params.split("-", 2)
       [score.to_f, jid]
     end
 
@@ -156,9 +192,13 @@ module Sidekiq
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
-      options = options.stringify_keys
+      # stringify
+      options.keys.each do |key|
+        options[key.to_s] = options.delete(key)
+      end
+
       params.merge(options).map do |key, value|
-        SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
+        SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       end.compact.join("&")
     end
 
@@ -246,6 +286,10 @@ module Sidekiq
       "Sidekiq v#{Sidekiq::VERSION}"
     end
 
+    def server_utc_time
+      Time.now.utc.strftime('%H:%M:%S UTC')
+    end
+
     def redis_connection_and_namespace
       @redis_connection_and_namespace ||= begin
         namespace_suffix = namespace == nil ? '' : "##{namespace}"

data/lib/sidekiq/web/router.rb

@@ -3,16 +3,16 @@ require 'rack'
 
 module Sidekiq
   module WebRouter
-    GET = 'GET'.freeze
-    DELETE = 'DELETE'.freeze
-    POST = 'POST'.freeze
-    PUT = 'PUT'.freeze
-    PATCH = 'PATCH'.freeze
-    HEAD = 'HEAD'.freeze
+    GET = 'GET'
+    DELETE = 'DELETE'
+    POST = 'POST'
+    PUT = 'PUT'
+    PATCH = 'PATCH'
+    HEAD = 'HEAD'
 
-    ROUTE_PARAMS = 'rack.route_params'.freeze
-    REQUEST_METHOD = 'REQUEST_METHOD'.freeze
-    PATH_INFO = 'PATH_INFO'.freeze
+    ROUTE_PARAMS = 'rack.route_params'
+    REQUEST_METHOD = 'REQUEST_METHOD'
+    PATH_INFO = 'PATH_INFO'
 
     def get(path, &block)
       route(GET, path, &block)
@@ -45,6 +45,10 @@ module Sidekiq
       request_method = env[REQUEST_METHOD]
       path_info = ::Rack::Utils.unescape env[PATH_INFO]
 
+      # There are servers which send an empty string when requesting the root.
+      # These servers should be ashamed of themselves.
+      path_info = "/" if path_info == ""
+
       @routes[request_method].each do |route|
         if params = route.match(request_method, path_info)
           env[ROUTE_PARAMS] = params
@@ -60,7 +64,7 @@ module Sidekiq
   class WebRoute
     attr_accessor :request_method, :pattern, :block, :name
 
-    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/.freeze
+    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/
 
     def initialize(request_method, pattern, block)
       @request_method = request_method