sidekiq 5.0.5 → 5.2.0

data/lib/sidekiq/redis_connection.rb

@@ -15,7 +15,15 @@ module Sidekiq
         options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{$$}" if !options.has_key?(:id)
         options[:url] ||= determine_redis_provider
 
-        size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
+        size = if options[:size]
+                 options[:size]
+               elsif Sidekiq.server?
+                 Sidekiq.options[:concurrency] + 5
+               elsif ENV['RAILS_MAX_THREADS']
+                 Integer(ENV['RAILS_MAX_THREADS'])
+               else
+                 5
+               end
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
@@ -38,7 +46,7 @@ module Sidekiq
       #   - enterprise's leader election
       #   - enterprise's cron support
       def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size <= concurrency
       end
 
       def build_client(options)
@@ -70,7 +78,7 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] ||= 'ruby'.freeze
+        opts[:driver] ||= 'ruby'
 
         # Issue #3303, redis-rb will silently retry an operation.
         # This can lead to duplicate jobs if Sidekiq::Client's LPUSH

data/lib/sidekiq/scheduled.rb

@@ -17,7 +17,7 @@ module Sidekiq
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
-            while job = conn.zrangebyscore(sorted_set, '-inf'.freeze, now, :limit => [0, 1]).first do
+            while job = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
 
               # Pop item off the queue and add it to the work queue. If the job can't be popped from
               # the queue, it's because another process already popped it so we can move on to the
@@ -79,9 +79,7 @@ module Sidekiq
           # Most likely a problem with redis networking.
           # Punt and try again at the next interval
           logger.error ex.message
-          ex.backtrace.each do |bt|
-            logger.error(bt)
-          end
+          handle_exception(ex)
         end
       end
 
@@ -95,13 +93,38 @@ module Sidekiq
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
         logger.error ex.message
-        logger.error ex.backtrace.first
+        handle_exception(ex)
         sleep 5
       end
 
-      # Calculates a random interval that is ±50% the desired average.
       def random_poll_interval
-        poll_interval_average * rand + poll_interval_average.to_f / 2
+        # We want one Sidekiq process to schedule jobs every N seconds.  We have M processes
+        # and **don't** want to coordinate.
+        #
+        # So in N*M second timespan, we want each process to schedule once.  The basic loop is:
+        #
+        # * sleep a random amount within that N*M timespan
+        # * wake up and schedule
+        #
+        # We want to avoid one edge case: imagine a set of 2 processes, scheduling every 5 seconds,
+        # so N*M = 10.  Each process decides to randomly sleep 8 seconds, now we've failed to meet
+        # that 5 second average. Thankfully each schedule cycle will sleep randomly so the next
+        # iteration could see each process sleep for 1 second, undercutting our average.
+        #
+        # So below 10 processes, we special case and ensure the processes sleep closer to the average.
+        # In the example above, each process should schedule every 10 seconds on average. We special
+        # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
+        # As we run more processes, the scheduling interval average will approach an even spread
+        # between 0 and poll interval so we don't need this artifical boost.
+        #
+        if process_count < 10
+          # For small clusters, calculate a random interval that is ±50% the desired average.
+          poll_interval_average * rand + poll_interval_average.to_f / 2
+        else
+          # With 10+ processes, we should have enough randomness to get decent polling
+          # across the entire timespan
+          poll_interval_average * rand
+        end
       end
 
       # We do our best to tune the poll interval to the size of the active Sidekiq
@@ -125,9 +148,13 @@ module Sidekiq
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
       def scaled_poll_interval
+        process_count * Sidekiq.options[:average_scheduled_poll_interval]
+      end
+
+      def process_count
         pcount = Sidekiq::ProcessSet.new.size
         pcount = 1 if pcount == 0
-        pcount * Sidekiq.options[:average_scheduled_poll_interval]
+        pcount
       end
 
       def initial_wait

data/lib/sidekiq/testing.rb

@@ -72,9 +72,7 @@ module Sidekiq
 
   class EmptyQueueError < RuntimeError; end
 
-  class Client
-    alias_method :raw_push_real, :raw_push
-
+  module TestingClient
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
@@ -92,11 +90,13 @@ module Sidekiq
         end
         true
       else
-        raw_push_real(payloads)
+        super
       end
     end
   end
 
+  Sidekiq::Client.prepend TestingClient
+
   module Queues
     ##
     # The Queues class is only for testing the fake queue implementation.

data/lib/sidekiq/util.rb

@@ -21,7 +21,7 @@ module Sidekiq
 
     def safe_thread(name, &block)
       Thread.new do
-        Thread.current['sidekiq_label'.freeze] = name
+        Thread.current['sidekiq_label'] = name
         watchdog(name, &block)
       end
     end
@@ -46,7 +46,10 @@ module Sidekiq
       @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
-    def fire_event(event, reverse=false)
+    def fire_event(event, options={})
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+
       arr = Sidekiq.options[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
@@ -54,6 +57,7 @@ module Sidekiq
           block.call
         rescue => ex
           handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
+          raise ex if reraise
         end
       end
       arr.clear

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "5.0.5"
+  VERSION = "5.2.0"
 end

data/lib/sidekiq/web/action.rb

@@ -2,7 +2,7 @@
 
 module Sidekiq
   class WebAction
-    RACK_SESSION = 'rack.session'.freeze
+    RACK_SESSION = 'rack.session'
 
     attr_accessor :env, :block, :type
 
@@ -77,7 +77,7 @@ module Sidekiq
     private
 
     def _erb(file, locals)
-      locals.each {|k, v| define_singleton_method(k){ v } } if locals
+      locals.each {|k, v| define_singleton_method(k){ v } unless (singleton_methods.include? k)} if locals
 
       if file.kind_of?(String)
         ERB.new(file).result(binding)

data/lib/sidekiq/web/application.rb

@@ -4,9 +4,24 @@ module Sidekiq
   class WebApplication
     extend WebRouter
 
-    CONTENT_LENGTH = "Content-Length".freeze
-    CONTENT_TYPE = "Content-Type".freeze
+    CONTENT_LENGTH = "Content-Length"
+    CONTENT_TYPE = "Content-Type"
     REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+    CSP_HEADER = [
+      "default-src 'self' https: http:",
+      "child-src 'self'",
+      "connect-src 'self' https: http: wss: ws:",
+      "font-src 'self' https: http:",
+      "frame-src 'self'",
+      "img-src 'self' https: http: data:",
+      "manifest-src 'self'",
+      "media-src 'self'",
+      "object-src 'none'",
+      "script-src 'self' https: http:",
+      "style-src 'self' https: http: 'unsafe-inline'",
+      "worker-src 'self'",
+      "base-uri 'self'"
+    ].join('; ').freeze
 
     def initialize(klass)
       @klass = klass
@@ -279,6 +294,7 @@ module Sidekiq
           "Content-Type" => "text/html",
           "Cache-Control" => "no-cache",
           "Content-Language" => action.locale,
+          "Content-Security-Policy" => CSP_HEADER
         }
 
         [200, headers, [resp]]

data/lib/sidekiq/web/helpers.rb

@@ -80,7 +80,7 @@ module Sidekiq
 
     # See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
     def user_preferred_languages
-      languages = env['HTTP_ACCEPT_LANGUAGE'.freeze]
+      languages = env['HTTP_ACCEPT_LANGUAGE']
       languages.to_s.downcase.gsub(/\s+/, '').split(',').map do |language|
         locale, quality = language.split(';q=', 2)
         locale  = nil if locale == '*' # Ignore wildcards
@@ -148,12 +148,14 @@ module Sidekiq
     end
 
     def redis_connection
-      attrs = Sidekiq.redis { |conn| conn.connection }
-      "redis://#{attrs[:location]}/#{attrs[:db]}"
+      Sidekiq.redis do |conn|
+        c = conn.connection
+        "redis://#{c[:location]}/#{c[:db]}"
+      end
     end
 
     def namespace
-      @@ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
+      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
     end
 
     def redis_info
@@ -182,7 +184,7 @@ module Sidekiq
     end
 
     def parse_params(params)
-      score, jid = params.split("-")
+      score, jid = params.split("-", 2)
       [score.to_f, jid]
     end
 

data/lib/sidekiq/web/router.rb

@@ -3,16 +3,16 @@ require 'rack'
 
 module Sidekiq
   module WebRouter
-    GET = 'GET'.freeze
-    DELETE = 'DELETE'.freeze
-    POST = 'POST'.freeze
-    PUT = 'PUT'.freeze
-    PATCH = 'PATCH'.freeze
-    HEAD = 'HEAD'.freeze
+    GET = 'GET'
+    DELETE = 'DELETE'
+    POST = 'POST'
+    PUT = 'PUT'
+    PATCH = 'PATCH'
+    HEAD = 'HEAD'
 
-    ROUTE_PARAMS = 'rack.route_params'.freeze
-    REQUEST_METHOD = 'REQUEST_METHOD'.freeze
-    PATH_INFO = 'PATH_INFO'.freeze
+    ROUTE_PARAMS = 'rack.route_params'
+    REQUEST_METHOD = 'REQUEST_METHOD'
+    PATH_INFO = 'PATH_INFO'
 
     def get(path, &block)
       route(GET, path, &block)
@@ -64,7 +64,7 @@ module Sidekiq
   class WebRoute
     attr_accessor :request_method, :pattern, :block, :name
 
-    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/.freeze
+    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/
 
     def initialize(request_method, pattern, block)
       @request_method = request_method

data/lib/sidekiq/web.rb

@@ -19,10 +19,10 @@ require 'rack/session/cookie'
 module Sidekiq
   class Web
     ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
-    VIEWS = "#{ROOT}/views".freeze
-    LOCALES = ["#{ROOT}/locales".freeze]
-    LAYOUT = "#{VIEWS}/layout.erb".freeze
-    ASSETS = "#{ROOT}/assets".freeze
+    VIEWS = "#{ROOT}/views"
+    LOCALES = ["#{ROOT}/locales"]
+    LAYOUT = "#{VIEWS}/layout.erb"
+    ASSETS = "#{ROOT}/assets"
 
     DEFAULT_TABS = {
       "Dashboard" => '',

data/lib/sidekiq/worker.rb

@@ -47,7 +47,7 @@ module Sidekiq
       end
 
       def perform_async(*args)
-        @klass.client_push(@opts.merge('args'.freeze => args, 'class'.freeze => @klass))
+        @klass.client_push(@opts.merge('args' => args, 'class' => @klass))
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -57,9 +57,9 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        payload = @opts.merge('class'.freeze => @klass, 'args'.freeze => args, 'at'.freeze => ts)
+        payload = @opts.merge('class' => @klass, 'args' => args, 'at' => ts)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        payload.delete('at'.freeze) if ts <= now
+        payload.delete('at') if ts <= now
         @klass.client_push(payload)
       end
       alias_method :perform_at, :perform_in
@@ -84,7 +84,7 @@ module Sidekiq
       end
 
       def perform_async(*args)
-        client_push('class'.freeze => self, 'args'.freeze => args)
+        client_push('class' => self, 'args' => args)
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -94,10 +94,10 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = { 'class'.freeze => self, 'args'.freeze => args, 'at'.freeze => ts }
+        item = { 'class' => self, 'args' => args, 'at' => ts }
 
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        item.delete('at'.freeze) if ts <= now
+        item.delete('at') if ts <= now
 
         client_push(item)
       end
@@ -134,7 +134,7 @@ module Sidekiq
       end
 
       def client_push(item) # :nodoc:
-        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'.freeze] || Sidekiq.redis_pool
+        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
         # stringify
         item.keys.each do |key|
           item[key.to_s] = item.delete(key)

data/lib/sidekiq.rb

@@ -1,7 +1,6 @@
-# encoding: utf-8
 # frozen_string_literal: true
 require 'sidekiq/version'
-fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.2.2." if RUBY_PLATFORM != 'java' && RUBY_VERSION < '2.2.2'
+fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.2.2." if RUBY_PLATFORM != 'java' && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2')
 
 require 'sidekiq/logging'
 require 'sidekiq/client'
@@ -12,19 +11,20 @@ require 'sidekiq/delay'
 require 'json'
 
 module Sidekiq
-  NAME = 'Sidekiq'.freeze
+  NAME = 'Sidekiq'
   LICENSE = 'See LICENSE and the LGPL-3.0 for licensing details.'
 
   DEFAULTS = {
     queues: [],
     labels: [],
-    concurrency: 25,
+    concurrency: 10,
     require: '.',
     environment: nil,
     timeout: 8,
     poll_interval_average: nil,
-    average_scheduled_poll_interval: 15,
+    average_scheduled_poll_interval: 5,
     error_handlers: [],
+    death_handlers: [],
     lifecycle_events: {
       startup: [],
       quiet: [],
@@ -47,7 +47,7 @@ module Sidekiq
     "connected_clients" => "9999",
     "used_memory_human" => "9P",
     "used_memory_peak_human" => "9P"
-  }.freeze
+  }
 
   def self.❨╯°□°❩╯︵┻━┻
     puts "Calm down, yo."
@@ -156,16 +156,23 @@ module Sidekiq
     defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
   end
 
+  def self.default_retries_exhausted=(prok)
+    logger.info { "default_retries_exhausted is deprecated, please use `config.death_handlers << -> {|job, ex| }`" }
+    return nil unless prok
+    death_handlers << prok
+  end
+
+  ##
+  # Death handlers are called when all retries for a job have been exhausted and
+  # the job dies.  It's the notification to your application
+  # that this job will not succeed without manual intervention.
+  #
   # Sidekiq.configure_server do |config|
-  #   config.default_retries_exhausted = -> (job, ex) do
+  #   config.death_handlers << ->(job, ex) do
   #   end
   # end
-  def self.default_retries_exhausted=(prok)
-    @default_retries_exhausted = prok
-  end
-  @default_retries_exhausted = ->(job, ex) { }
-  def self.default_retries_exhausted
-    @default_retries_exhausted
+  def self.death_handlers
+    options[:death_handlers]
   end
 
   def self.load_json(string)

data/sidekiq.gemspec

@@ -17,12 +17,7 @@ Gem::Specification.new do |gem|
   gem.version       = Sidekiq::VERSION
   gem.required_ruby_version = ">= 2.2.2"
 
-  gem.add_dependency                  'redis', '>= 3.3.4', '< 5'
-  gem.add_dependency                  'connection_pool', '~> 2.2', '>= 2.2.0'
-  gem.add_dependency                  'concurrent-ruby', '~> 1.0'
-  gem.add_dependency                  'rack-protection', '>= 1.5.0'
-  gem.add_development_dependency      'redis-namespace', '~> 1.5', '>= 1.5.2'
-  gem.add_development_dependency      'minitest', '~> 5.10', '>= 5.10.1'
-  gem.add_development_dependency      'rake', '~> 10.0'
-  gem.add_development_dependency      'rails', '>= 3.2.0'
+  gem.add_dependency 'redis', '>= 3.3.5', '< 5'
+  gem.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
+  gem.add_dependency 'rack-protection', '>= 1.5.0'
 end