sidekiq 4.2.4 → 5.2.10

data/lib/sidekiq/scheduled.rb

@@ -17,7 +17,7 @@ module Sidekiq
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
-            while job = conn.zrangebyscore(sorted_set, '-inf'.freeze, now, :limit => [0, 1]).first do
+            while job = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
 
               # Pop item off the queue and add it to the work queue. If the job can't be popped from
               # the queue, it's because another process already popped it so we can move on to the
@@ -79,9 +79,7 @@ module Sidekiq
           # Most likely a problem with redis networking.
           # Punt and try again at the next interval
           logger.error ex.message
-          ex.backtrace.each do |bt|
-            logger.error(bt)
-          end
+          handle_exception(ex)
         end
       end
 
@@ -95,13 +93,38 @@ module Sidekiq
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
         logger.error ex.message
-        logger.error ex.backtrace.first
+        handle_exception(ex)
         sleep 5
       end
 
-      # Calculates a random interval that is ±50% the desired average.
       def random_poll_interval
-        poll_interval_average * rand + poll_interval_average.to_f / 2
+        # We want one Sidekiq process to schedule jobs every N seconds.  We have M processes
+        # and **don't** want to coordinate.
+        #
+        # So in N*M second timespan, we want each process to schedule once.  The basic loop is:
+        #
+        # * sleep a random amount within that N*M timespan
+        # * wake up and schedule
+        #
+        # We want to avoid one edge case: imagine a set of 2 processes, scheduling every 5 seconds,
+        # so N*M = 10.  Each process decides to randomly sleep 8 seconds, now we've failed to meet
+        # that 5 second average. Thankfully each schedule cycle will sleep randomly so the next
+        # iteration could see each process sleep for 1 second, undercutting our average.
+        #
+        # So below 10 processes, we special case and ensure the processes sleep closer to the average.
+        # In the example above, each process should schedule every 10 seconds on average. We special
+        # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
+        # As we run more processes, the scheduling interval average will approach an even spread
+        # between 0 and poll interval so we don't need this artifical boost.
+        #
+        if process_count < 10
+          # For small clusters, calculate a random interval that is ±50% the desired average.
+          poll_interval_average * rand + poll_interval_average.to_f / 2
+        else
+          # With 10+ processes, we should have enough randomness to get decent polling
+          # across the entire timespan
+          poll_interval_average * rand
+        end
       end
 
       # We do our best to tune the poll interval to the size of the active Sidekiq
@@ -125,9 +148,13 @@ module Sidekiq
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
       def scaled_poll_interval
+        process_count * Sidekiq.options[:average_scheduled_poll_interval]
+      end
+
+      def process_count
         pcount = Sidekiq::ProcessSet.new.size
         pcount = 1 if pcount == 0
-        pcount * Sidekiq.options[:average_scheduled_poll_interval]
+        pcount
       end
 
       def initial_wait

data/lib/sidekiq/testing.rb

@@ -55,6 +55,15 @@ module Sidekiq
         yield @server_chain if block_given?
         @server_chain
       end
+
+      def constantize(str)
+        names = str.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      end
     end
   end
 
@@ -63,29 +72,31 @@ module Sidekiq
 
   class EmptyQueueError < RuntimeError; end
 
-  class Client
-    alias_method :raw_push_real, :raw_push
-
+  module TestingClient
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          Queues.push(job['queue'], job['class'], Sidekiq.load_json(Sidekiq.dump_json(job)))
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
+          Queues.push(job['queue'], job['class'], job)
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = job['class'].constantize
+          klass = Sidekiq::Testing.constantize(job['class'])
           job['id'] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
         end
         true
       else
-        raw_push_real(payloads)
+        super
       end
     end
   end
 
+  Sidekiq::Client.prepend TestingClient
+
   module Queues
     ##
     # The Queues class is only for testing the fake queue implementation.
@@ -307,10 +318,16 @@ module Sidekiq
           worker_classes = jobs.map { |job| job["class"] }.uniq
 
           worker_classes.each do |worker_class|
-            worker_class.constantize.drain
+            Sidekiq::Testing.constantize(worker_class).drain
           end
         end
       end
     end
   end
 end
+
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
+  puts("**************************************************")
+  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
+  puts("**************************************************")
+end

data/lib/sidekiq/util.rb

@@ -2,7 +2,6 @@
 require 'socket'
 require 'securerandom'
 require 'sidekiq/exception_handler'
-require 'sidekiq/core_ext'
 
 module Sidekiq
   ##
@@ -22,6 +21,7 @@ module Sidekiq
 
     def safe_thread(name, &block)
       Thread.new do
+        Thread.current['sidekiq_label'] = name
         watchdog(name, &block)
       end
     end
@@ -46,7 +46,10 @@ module Sidekiq
       @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
-    def fire_event(event, reverse=false)
+    def fire_event(event, options={})
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+
       arr = Sidekiq.options[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
@@ -54,6 +57,7 @@ module Sidekiq
           block.call
         rescue => ex
           handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
+          raise ex if reraise
         end
       end
       arr.clear

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "4.2.4"
+  VERSION = "5.2.10"
 end

data/lib/sidekiq/web/action.rb

@@ -2,7 +2,7 @@
 
 module Sidekiq
   class WebAction
-    RACK_SESSION = 'rack.session'.freeze
+    RACK_SESSION = 'rack.session'
 
     attr_accessor :env, :block, :type
 
@@ -15,7 +15,7 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, res
+      throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
@@ -39,10 +39,6 @@ module Sidekiq
       env[RACK_SESSION]
     end
 
-    def content_type(type)
-      @type = type
-    end
-
     def erb(content, options = {})
       if content.kind_of? Symbol
         unless respond_to?(:"_erb_#{content}")
@@ -81,7 +77,7 @@ module Sidekiq
     private
 
     def _erb(file, locals)
-      locals.each {|k, v| define_singleton_method(k){ v } } if locals
+      locals.each {|k, v| define_singleton_method(k){ v } unless (singleton_methods.include? k)} if locals
 
       if file.kind_of?(String)
         ERB.new(file).result(binding)

data/lib/sidekiq/web/application.rb

@@ -4,9 +4,24 @@ module Sidekiq
   class WebApplication
     extend WebRouter
 
-    CONTENT_LENGTH = "Content-Length".freeze
-    CONTENT_TYPE = "Content-Type".freeze
+    CONTENT_LENGTH = "Content-Length"
+    CONTENT_TYPE = "Content-Type"
     REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+    CSP_HEADER = [
+      "default-src 'self' https: http:",
+      "child-src 'self'",
+      "connect-src 'self' https: http: wss: ws:",
+      "font-src 'self' https: http:",
+      "frame-src 'self'",
+      "img-src 'self' https: http: data:",
+      "manifest-src 'self'",
+      "media-src 'self'",
+      "object-src 'none'",
+      "script-src 'self' https: http: 'unsafe-inline'",
+      "style-src 'self' https: http: 'unsafe-inline'",
+      "worker-src 'self'",
+      "base-uri 'self'"
+    ].join('; ').freeze
 
     def initialize(klass)
       @klass = klass
@@ -28,13 +43,12 @@ module Sidekiq
       # nothing, backwards compatibility
     end
 
-    get "" do
-      redirect(root_path)
-    end
-
     get "/" do
       @redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
-      stats_history = Sidekiq::Stats::History.new((params['days'] || 30).to_i)
+      days = (params["days"] || 30).to_i
+      return halt(401) if days < 1 || days > 180
+
+      stats_history = Sidekiq::Stats::History.new(days)
       @processed_history = stats_history.processed
       @failed_history = stats_history.failed
 
@@ -89,7 +103,7 @@ module Sidekiq
       name = route_params[:name]
       Sidekiq::Job.new(params['key_val'], name).delete
 
-      redirect_with_query("#{root_path}queues/#{name}")
+      redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
     end
 
     get '/morgue' do
@@ -185,6 +199,12 @@ module Sidekiq
       redirect "#{root_path}retries"
     end
 
+    post "/retries/all/kill" do
+      Sidekiq::RetrySet.new.kill_all
+
+      redirect "#{root_path}retries"
+    end
+
     post "/retries/:key" do
       job = Sidekiq::RetrySet.new.fetch(*parse_params(route_params[:key])).first
 
@@ -238,7 +258,6 @@ module Sidekiq
     get '/stats' do
       sidekiq_stats = Sidekiq::Stats.new
       redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
-
       json(
         sidekiq: {
           processed:       sidekiq_stats.processed,
@@ -251,7 +270,8 @@ module Sidekiq
           dead:            sidekiq_stats.dead_size,
           default_latency: sidekiq_stats.default_queue_latency
         },
-        redis: redis_stats
+        redis: redis_stats,
+        server_utc_time: server_utc_time
       )
     end
 
@@ -278,19 +298,15 @@ module Sidekiq
       resp = case resp
       when Array
         resp
-      when Fixnum
-        [resp, {}, []]
       else
-        type_header = case action.type
-        when :json
-          { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }
-        when String
-          { "Content-Type" => action.type, "Cache-Control" => "no-cache" }
-        else
-          { "Content-Type" => "text/html", "Cache-Control" => "no-cache" }
-        end
-
-        [200, type_header, [resp]]
+        headers = {
+          "Content-Type" => "text/html",
+          "Cache-Control" => "no-cache",
+          "Content-Language" => action.locale,
+          "Content-Security-Policy" => CSP_HEADER
+        }
+
+        [200, headers, [resp]]
       end
 
       resp[1] = resp[1].dup

data/lib/sidekiq/web/helpers.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 require 'uri'
+require 'set'
 require 'yaml'
+require 'cgi'
 
 module Sidekiq
   # This is not a public API
@@ -13,7 +15,7 @@ module Sidekiq
         settings.locales.each_with_object({}) do |path, global|
           find_locale_files(lang).each do |file|
             strs = YAML.load(File.open(file))
-            global.deep_merge!(strs[lang])
+            global.merge!(strs[lang])
           end
         end
       end
@@ -22,6 +24,7 @@ module Sidekiq
     def clear_caches
       @@strings = nil
       @@locale_files = nil
+      @@available_locales = nil
     end
 
     def locale_files
@@ -30,6 +33,10 @@ module Sidekiq
       end
     end
 
+    def available_locales
+      @@available_locales ||= locale_files.map { |path| File.basename(path, '.yml') }.uniq
+    end
+
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
@@ -63,29 +70,58 @@ module Sidekiq
       end
     end
 
-    # Given a browser request Accept-Language header like
-    # "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2", this function
-    # will return "fr" since that's the first code with a matching
-    # locale in web/locales
+    def text_direction
+      get_locale['TextDirection'] || 'ltr'
+    end
+
+    def rtl?
+      text_direction == 'rtl'
+    end
+
+    # See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+    def user_preferred_languages
+      languages = env['HTTP_ACCEPT_LANGUAGE']
+      languages.to_s.downcase.gsub(/\s+/, '').split(',').map do |language|
+        locale, quality = language.split(';q=', 2)
+        locale  = nil if locale == '*' # Ignore wildcards
+        quality = quality ? quality.to_f : 1.0
+        [locale, quality]
+      end.sort do |(_, left), (_, right)|
+        right <=> left
+      end.map(&:first).compact
+    end
+
+    # Given an Accept-Language header like "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2"
+    # this method will try to best match the available locales to the user's preferred languages.
+    #
+    # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
       @locale ||= begin
-        locale = 'en'.freeze
-        languages = env['HTTP_ACCEPT_LANGUAGE'.freeze] || 'en'.freeze
-        languages.downcase.split(','.freeze).each do |lang|
-          next if lang == '*'.freeze
-          lang = lang.split(';'.freeze)[0]
-          break locale = lang if find_locale_files(lang).any?
-        end
-        locale
+        matched_locale = user_preferred_languages.map do |preferred|
+          preferred_language = preferred.split('-', 2).first
+
+          lang_group = available_locales.select do |available|
+            preferred_language == available.split('-', 2).first
+          end
+
+          lang_group.find { |lang| lang == preferred } || lang_group.min_by(&:length)
+        end.compact.first
+
+        matched_locale || 'en'
       end
     end
 
+    # mperham/sidekiq#3243
+    def unfiltered?
+      yield unless env['PATH_INFO'].start_with?("/filter/")
+    end
+
     def get_locale
       strings(locale)
     end
 
     def t(msg, options={})
-      string = get_locale[msg] || msg
+      string = get_locale[msg] || strings('en')[msg] || msg
       if options.empty?
         string
       else
@@ -111,16 +147,15 @@ module Sidekiq
       end.map { |msg| Sidekiq.load_json(msg) }
     end
 
-    def location
-      Sidekiq.redis { |conn| conn.client.location }
-    end
-
     def redis_connection
-      Sidekiq.redis { |conn| conn.client.id }
+      Sidekiq.redis do |conn|
+        c = conn.connection
+        "redis://#{c[:location]}/#{c[:db]}"
+      end
     end
 
     def namespace
-      @@ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
+      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
     end
 
     def redis_info
@@ -140,7 +175,8 @@ module Sidekiq
     end
 
     def relative_time(time)
-      %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
+      stamp = time.getutc.iso8601
+      %{<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>}
     end
 
     def job_params(job, score)
@@ -148,7 +184,7 @@ module Sidekiq
     end
 
     def parse_params(params)
-      score, jid = params.split("-")
+      score, jid = params.split("-", 2)
       [score.to_f, jid]
     end
 
@@ -156,9 +192,13 @@ module Sidekiq
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
-      options = options.stringify_keys
+      # stringify
+      options.keys.each do |key|
+        options[key.to_s] = options.delete(key)
+      end
+
       params.merge(options).map do |key, value|
-        SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
+        SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       end.compact.join("&")
     end
 
@@ -167,9 +207,16 @@ module Sidekiq
     end
 
     def display_args(args, truncate_after_chars = 2000)
-      args.map do |arg|
-        h(truncate(to_display(arg), truncate_after_chars))
-      end.join(", ")
+      return "Invalid job payload, args is nil" if args == nil
+      return "Invalid job payload, args must be an Array, not #{args.class.name}" if !args.is_a?(Array)
+
+      begin
+        args.map do |arg|
+          h(truncate(to_display(arg), truncate_after_chars))
+        end.join(", ")
+      rescue
+        "Illegal job arguments: #{h args.inspect}"
+      end
     end
 
     def csrf_tag
@@ -246,6 +293,10 @@ module Sidekiq
       "Sidekiq v#{Sidekiq::VERSION}"
     end
 
+    def server_utc_time
+      Time.now.utc.strftime('%H:%M:%S UTC')
+    end
+
     def redis_connection_and_namespace
       @redis_connection_and_namespace ||= begin
         namespace_suffix = namespace == nil ? '' : "##{namespace}"

data/lib/sidekiq/web/router.rb

@@ -3,16 +3,16 @@ require 'rack'
 
 module Sidekiq
   module WebRouter
-    GET = 'GET'.freeze
-    DELETE = 'DELETE'.freeze
-    POST = 'POST'.freeze
-    PUT = 'PUT'.freeze
-    PATCH = 'PATCH'.freeze
-    HEAD = 'HEAD'.freeze
+    GET = 'GET'
+    DELETE = 'DELETE'
+    POST = 'POST'
+    PUT = 'PUT'
+    PATCH = 'PATCH'
+    HEAD = 'HEAD'
 
-    ROUTE_PARAMS = 'rack.route_params'.freeze
-    REQUEST_METHOD = 'REQUEST_METHOD'.freeze
-    PATH_INFO = 'PATH_INFO'.freeze
+    ROUTE_PARAMS = 'rack.route_params'
+    REQUEST_METHOD = 'REQUEST_METHOD'
+    PATH_INFO = 'PATH_INFO'
 
     def get(path, &block)
       route(GET, path, &block)
@@ -45,6 +45,10 @@ module Sidekiq
       request_method = env[REQUEST_METHOD]
       path_info = ::Rack::Utils.unescape env[PATH_INFO]
 
+      # There are servers which send an empty string when requesting the root.
+      # These servers should be ashamed of themselves.
+      path_info = "/" if path_info == ""
+
       @routes[request_method].each do |route|
         if params = route.match(request_method, path_info)
           env[ROUTE_PARAMS] = params
@@ -60,7 +64,7 @@ module Sidekiq
   class WebRoute
     attr_accessor :request_method, :pattern, :block, :name
 
-    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/.freeze
+    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/
 
     def initialize(request_method, pattern, block)
       @request_method = request_method

data/lib/sidekiq/web.rb

@@ -19,10 +19,10 @@ require 'rack/session/cookie'
 module Sidekiq
   class Web
     ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
-    VIEWS = "#{ROOT}/views".freeze
-    LOCALES = ["#{ROOT}/locales".freeze]
-    LAYOUT = "#{VIEWS}/layout.erb".freeze
-    ASSETS = "#{ROOT}/assets".freeze
+    VIEWS = "#{ROOT}/views"
+    LOCALES = ["#{ROOT}/locales"]
+    LAYOUT = "#{VIEWS}/layout.erb"
+    ASSETS = "#{ROOT}/assets"
 
     DEFAULT_TABS = {
       "Dashboard" => '',