sidekiq 6.4.2 → 6.5.2

data/lib/sidekiq/redis_connection.rb

@@ -5,8 +5,81 @@ require "redis"
 require "uri"
 
 module Sidekiq
-  class RedisConnection
+  module RedisConnection
+    class RedisAdapter
+      BaseError = Redis::BaseError
+      CommandError = Redis::CommandError
+
+      def initialize(options)
+        warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
+        @options = options
+      end
+
+      def new_client
+        namespace = @options[:namespace]
+
+        client = Redis.new client_opts(@options)
+        if namespace
+          begin
+            require "redis/namespace"
+            Redis::Namespace.new(namespace, redis: client)
+          rescue LoadError
+            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
+            exit(-127)
+          end
+        else
+          client
+        end
+      end
+
+      private
+
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
+        end
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
+
+        opts
+      end
+    end
+
+    @adapter = RedisAdapter
+
     class << self
+      attr_reader :adapter
+
+      # RedisConnection.adapter = :redis
+      # RedisConnection.adapter = :redis_client
+      def adapter=(adapter)
+        raise "no" if adapter == self
+        result = case adapter
+        when :redis
+          RedisAdapter
+        when Class
+          adapter
+        else
+          require "sidekiq/#{adapter}_adapter"
+          nil
+        end
+        @adapter = result if result
+      end
+
       def create(options = {})
         symbolized_options = options.transform_keys(&:to_sym)
 
@@ -19,20 +92,21 @@ module Sidekiq
         elsif Sidekiq.server?
           # Give ourselves plenty of connections.  pool is lazy
           # so we won't create them until we need them.
-          Sidekiq.options[:concurrency] + 5
+          Sidekiq[:concurrency] + 5
         elsif ENV["RAILS_MAX_THREADS"]
           Integer(ENV["RAILS_MAX_THREADS"])
         else
           5
         end
 
-        verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
+        verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
 
         pool_timeout = symbolized_options[:pool_timeout] || 1
         log_info(symbolized_options)
 
+        redis_config = adapter.new(symbolized_options)
         ConnectionPool.new(timeout: pool_timeout, size: size) do
-          build_client(symbolized_options)
+          redis_config.new_client
         end
       end
 
@@ -50,47 +124,6 @@ module Sidekiq
         raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
       end
 
-      def build_client(options)
-        namespace = options[:namespace]
-
-        client = Redis.new client_opts(options)
-        if namespace
-          begin
-            require "redis/namespace"
-            Redis::Namespace.new(namespace, redis: client)
-          rescue LoadError
-            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
-                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
-            exit(-127)
-          end
-        else
-          client
-        end
-      end
-
-      def client_opts(options)
-        opts = options.dup
-        if opts[:namespace]
-          opts.delete(:namespace)
-        end
-
-        if opts[:network_timeout]
-          opts[:timeout] = opts[:network_timeout]
-          opts.delete(:network_timeout)
-        end
-
-        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
-
-        # Issue #3303, redis-rb will silently retry an operation.
-        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
-        # is performed twice but I believe this is much, much rarer
-        # than the reconnect silently fixing a problem; we keep it
-        # on by default.
-        opts[:reconnect_attempts] ||= 1
-
-        opts
-      end
-
       def log_info(options)
         redacted = "REDACTED"
 
@@ -110,9 +143,9 @@ module Sidekiq
           sentinel[:password] = redacted if sentinel[:password]
         end
         if Sidekiq.server?
-          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
+          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
         else
-          Sidekiq.logger.debug("#{Sidekiq::NAME} client with redis options #{scrubbed_options}")
+          Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
         end
       end
 

data/lib/sidekiq/ring_buffer.rb

@@ -0,0 +1,29 @@
+require "forwardable"
+
+module Sidekiq
+  class RingBuffer
+    include Enumerable
+    extend Forwardable
+    def_delegators :@buf, :[], :each, :size
+
+    def initialize(size, default = 0)
+      @size = size
+      @buf = Array.new(size, default)
+      @index = 0
+    end
+
+    def <<(element)
+      @buf[@index % @size] = element
+      @index += 1
+      element
+    end
+
+    def buffer
+      @buf
+    end
+
+    def reset(default = 0)
+      @buf.fill(default)
+    end
+  end
+end

data/lib/sidekiq/scheduled.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require "sidekiq"
-require "sidekiq/util"
 require "sidekiq/api"
+require "sidekiq/component"
 
 module Sidekiq
   module Scheduled
@@ -52,8 +52,8 @@ module Sidekiq
           @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
-        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
-      rescue Redis::CommandError => e
+        conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
+      rescue RedisConnection.adapter::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
         @lua_zpopbyscore_sha = nil
@@ -67,12 +67,13 @@ module Sidekiq
     # just pops the job back onto its original queue so the
     # workers can pick it up like any other job.
     class Poller
-      include Util
+      include Sidekiq::Component
 
       INITIAL_WAIT = 10
 
-      def initialize
-        @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+      def initialize(options)
+        @config = options
+        @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
@@ -100,7 +101,7 @@ module Sidekiq
             enqueue
             wait
           end
-          Sidekiq.logger.info("Scheduler exiting...")
+          logger.info("Scheduler exiting...")
         }
       end
 
@@ -171,14 +172,14 @@ module Sidekiq
       #
       # We only do this if poll_interval_average is unset (the default).
       def poll_interval_average
-        Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
+        @config[:poll_interval_average] ||= scaled_poll_interval
       end
 
       # Calculates an average poll interval based on the number of known Sidekiq processes.
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
       def scaled_poll_interval
-        process_count * Sidekiq.options[:average_scheduled_poll_interval]
+        process_count * @config[:average_scheduled_poll_interval]
       end
 
       def process_count
@@ -197,7 +198,7 @@ module Sidekiq
         # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
         # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
         total = 0
-        total += INITIAL_WAIT unless Sidekiq.options[:poll_interval_average]
+        total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
         @sleeper.pop(total)

data/lib/sidekiq/testing.rb

@@ -188,7 +188,7 @@ module Sidekiq
       end
 
       def clear_for(queue, klass)
-        jobs_by_queue[queue].clear
+        jobs_by_queue[queue.to_s].clear
         jobs_by_class[klass].clear
       end
 

data/lib/sidekiq/transaction_aware_client.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq/client"
+
+module Sidekiq
+  class TransactionAwareClient
+    def initialize(redis_pool)
+      @redis_client = Client.new(redis_pool)
+    end
+
+    def push(item)
+      # pre-allocate the JID so we can return it immediately and
+      # save it to the database as part of the transaction.
+      item["jid"] ||= SecureRandom.hex(12)
+      AfterCommitEverywhere.after_commit { @redis_client.push(item) }
+      item["jid"]
+    end
+
+    ##
+    # We don't provide transactionality for push_bulk because we don't want
+    # to hold potentially hundreds of thousands of job records in memory due to
+    # a long running enqueue process.
+    def push_bulk(items)
+      @redis_client.push_bulk(items)
+    end
+  end
+end
+
+##
+# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
+module Sidekiq
+  def self.transactional_push!
+    begin
+      require "after_commit_everywhere"
+    rescue LoadError
+      Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
+      raise
+    end
+
+    default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
+    true
+  end
+end

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.4.2"
+  VERSION = "6.5.2"
 end

data/lib/sidekiq/web/application.rb

@@ -60,6 +60,19 @@ module Sidekiq
       erb(:dashboard)
     end
 
+    get "/metrics" do
+      q = Sidekiq::Metrics::Query.new
+      @resultset = q.top_jobs
+      erb(:metrics)
+    end
+
+    get "/metrics/:name" do
+      @name = route_params[:name]
+      q = Sidekiq::Metrics::Query.new
+      @resultset = q.for_job(@name)
+      erb(:metrics_for_job)
+    end
+
     get "/busy" do
       erb(:busy)
     end

data/lib/sidekiq/web/helpers.rb

@@ -15,7 +15,7 @@ module Sidekiq
       # so extensions can be localized
       @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.load(File.open(file))
+          strs = YAML.safe_load(File.open(file))
           global.merge!(strs[lang])
         end
       end
@@ -148,6 +148,29 @@ module Sidekiq
       @processes ||= Sidekiq::ProcessSet.new
     end
 
+    # Sorts processes by hostname following the natural sort order so that
+    # 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
+    # '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
+    def sorted_processes
+      @sorted_processes ||= begin
+        return processes unless processes.all? { |p| p["hostname"] }
+
+        split_characters = /[._-]/
+
+        padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
+
+        processes.to_a.sort_by do |process|
+          process["hostname"].split(split_characters).map do |substring|
+            # Left-pad the substring with '0' if it starts with a number or 'a'
+            # otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
+            padding_char = substring[0].match?(/\d/) ? "0" : "a"
+
+            substring.rjust(padding, padding_char)
+          end
+        end
+      end
+    end
+
     def stats
       @stats ||= Sidekiq::Stats.new
     end
@@ -301,7 +324,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end

data/lib/sidekiq/web.rb

@@ -33,6 +33,10 @@ module Sidekiq
       "Dead" => "morgue"
     }
 
+    if ENV["SIDEKIQ_METRICS_BETA"] == "1"
+      DEFAULT_TABS["Metrics"] = "metrics"
+    end
+
     class << self
       def settings
         self

data/lib/sidekiq/worker.rb

@@ -359,7 +359,8 @@ module Sidekiq
 
       def build_client # :nodoc:
         pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
-        Sidekiq::Client.new(pool)
+        client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
+        client_class.new(pool)
       end
     end
   end

data/lib/sidekiq.rb

@@ -5,6 +5,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
 
 require "sidekiq/logger"
 require "sidekiq/client"
+require "sidekiq/transaction_aware_client"
 require "sidekiq/worker"
 require "sidekiq/job"
 require "sidekiq/redis_connection"
@@ -33,7 +34,10 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
-      heartbeat: []
+      # triggers when we fire the first heartbeat on startup OR repairing a network partition
+      heartbeat: [],
+      # triggers on EVERY heartbeat call, every 10 seconds
+      beat: []
     },
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
@@ -52,19 +56,84 @@ module Sidekiq
     puts "Calm down, yo."
   end
 
+  # config.concurrency = 5
+  def self.concurrency=(val)
+    self[:concurrency] = Integer(val)
+  end
+
+  # config.queues = %w( high default low )                 # strict
+  # config.queues = %w( high,3 default,2 low,1 )           # weighted
+  # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
+  #
+  # With weighted priority, queue will be checked first (weight / total) of the time.
+  # high will be checked first (3/6) or 50% of the time.
+  # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
+  # are ridiculous and unnecessarily expensive. You can get random queue ordering
+  # by explicitly setting all weights to 1.
+  def self.queues=(val)
+    self[:queues] = Array(val).each_with_object([]) do |qstr, memo|
+      name, weight = qstr.split(",")
+      self[:strict] = false if weight.to_i > 0
+      [weight.to_i, 1].max.times do
+        memo << name
+      end
+    end
+  end
+
+  ### Private APIs
+  def self.default_error_handler(ex, ctx)
+    logger.warn(dump_json(ctx)) unless ctx.empty?
+    logger.warn("#{ex.class.name}: #{ex.message}")
+    logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
+  end
+
+  # DEFAULT_ERROR_HANDLER is a constant that allows the default error handler to
+  # be referenced. It must be defined here, after the default_error_handler
+  # method is defined.
+  DEFAULT_ERROR_HANDLER = method(:default_error_handler)
+
+  @config = DEFAULTS.dup
   def self.options
-    @options ||= DEFAULTS.dup
+    logger.warn "`config.options[:key] = value` is deprecated, use `config[:key] = value`: #{caller(1..2)}"
+    @config
   end
 
   def self.options=(opts)
-    @options = opts
+    logger.warn "config.options = hash` is deprecated, use `config.merge!(hash)`: #{caller(1..2)}"
+    @config = opts
+  end
+
+  def self.[](key)
+    @config[key]
+  end
+
+  def self.[]=(key, val)
+    @config[key] = val
+  end
+
+  def self.merge!(hash)
+    @config.merge!(hash)
+  end
+
+  def self.fetch(*args, &block)
+    @config.fetch(*args, &block)
+  end
+
+  def self.handle_exception(ex, ctx = {})
+    self[:error_handlers].each do |handler|
+      handler.call(ex, ctx)
+    rescue => ex
+      logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+      logger.error ex
+      logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+    end
   end
+  ###
 
   ##
   # Configuration for Sidekiq server, use like:
   #
   #   Sidekiq.configure_server do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
   #     config.server_middleware do |chain|
   #       chain.add MyServerHook
   #     end
@@ -77,7 +146,7 @@ module Sidekiq
   # Configuration for Sidekiq client, use like:
   #
   #   Sidekiq.configure_client do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/0' }
+  #     config.redis = { size: 1, url: 'redis://myhost:8877/0' }
   #   end
   def self.configure_client
     yield self unless server?
@@ -93,7 +162,7 @@ module Sidekiq
       retryable = true
       begin
         yield conn
-      rescue Redis::BaseError => ex
+      rescue RedisConnection.adapter::BaseError => ex
         # 2550 Failover can cause the server to become a replica, need
         # to disconnect and reopen the socket to get back to the primary.
         # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
@@ -118,7 +187,7 @@ module Sidekiq
       else
         conn.info
       end
-    rescue Redis::CommandError => ex
+    rescue RedisConnection.adapter::CommandError => ex
       # 2850 return fake version when INFO command has (probably) been renamed
       raise unless /unknown command/.match?(ex.message)
       FAKE_INFO
@@ -126,19 +195,19 @@ module Sidekiq
   end
 
   def self.redis_pool
-    @redis ||= Sidekiq::RedisConnection.create
+    @redis ||= RedisConnection.create
   end
 
   def self.redis=(hash)
     @redis = if hash.is_a?(ConnectionPool)
       hash
     else
-      Sidekiq::RedisConnection.create(hash)
+      RedisConnection.create(hash)
     end
   end
 
   def self.client_middleware
-    @client_chain ||= Middleware::Chain.new
+    @client_chain ||= Middleware::Chain.new(self)
     yield @client_chain if block_given?
     @client_chain
   end
@@ -150,7 +219,7 @@ module Sidekiq
   end
 
   def self.default_server_middleware
-    Middleware::Chain.new
+    Middleware::Chain.new(self)
   end
 
   def self.default_worker_options=(hash) # deprecated
@@ -179,7 +248,7 @@ module Sidekiq
   #   end
   # end
   def self.death_handlers
-    options[:death_handlers]
+    self[:death_handlers]
   end
 
   def self.load_json(string)
@@ -209,7 +278,7 @@ module Sidekiq
 
   def self.logger=(logger)
     if logger.nil?
-      self.logger.fatal!
+      self.logger.level = Logger::FATAL
       return self.logger
     end
 
@@ -232,7 +301,7 @@ module Sidekiq
   #
   # See sidekiq/scheduled.rb for an in-depth explanation of this value
   def self.average_scheduled_poll_interval=(interval)
-    options[:average_scheduled_poll_interval] = interval
+    self[:average_scheduled_poll_interval] = interval
   end
 
   # Register a proc to handle any error which occurs within the Sidekiq process.
@@ -243,7 +312,7 @@ module Sidekiq
   #
   # The default error handler logs errors to Sidekiq.logger.
   def self.error_handlers
-    options[:error_handlers]
+    self[:error_handlers]
   end
 
   # Register a block to run at a point in the Sidekiq lifecycle.
@@ -256,12 +325,12 @@ module Sidekiq
   #   end
   def self.on(event, &block)
     raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-    raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
-    options[:lifecycle_events][event] << block
+    raise ArgumentError, "Invalid event name: #{event}" unless self[:lifecycle_events].key?(event)
+    self[:lifecycle_events][event] << block
   end
 
   def self.strict_args!(mode = :raise)
-    options[:on_complex_arguments] = mode
+    self[:on_complex_arguments] = mode
   end
 
   # We are shutting down Sidekiq but what about threads that

data/sidekiq.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
     "source_code_uri" => "https://github.com/mperham/sidekiq"
   }
 
-  gem.add_dependency "redis", ">= 4.2.0"
+  gem.add_dependency "redis", ">= 4.5.0"
   gem.add_dependency "connection_pool", ">= 2.2.2"
   gem.add_dependency "rack", "~> 2.0"
 end

data/web/assets/javascripts/application.js

@@ -63,7 +63,7 @@ function addPollingListeners(_event)  {
 function addDataToggleListeners(event) {
   var source = event.target || event.srcElement;
   var targName = source.getAttribute("data-toggle");
-  var full = document.getElementById(targName + "_full");
+  var full = document.getElementById(targName);
   if (full.style.display == "block") {
     full.style.display = 'none';
   } else {