sidekiq 6.5.1 → 6.5.6

data/lib/sidekiq/middleware/chain.rb

@@ -4,84 +4,98 @@ require "sidekiq/middleware/modules"
 
 module Sidekiq
   # Middleware is code configured to run before/after
-  # a message is processed.  It is patterned after Rack
+  # a job is processed.  It is patterned after Rack
   # middleware. Middleware exists for the client side
   # (pushing jobs onto the queue) as well as the server
   # side (when jobs are actually processed).
   #
+  # Callers will register middleware Classes and Sidekiq will
+  # create new instances of the middleware for every job. This
+  # is important so that instance state is not shared accidentally
+  # between job executions.
+  #
   # To add middleware for the client:
   #
-  # Sidekiq.configure_client do |config|
-  #   config.client_middleware do |chain|
-  #     chain.add MyClientHook
+  #   Sidekiq.configure_client do |config|
+  #     config.client_middleware do |chain|
+  #       chain.add MyClientHook
+  #     end
   #   end
-  # end
   #
   # To modify middleware for the server, just call
   # with another block:
   #
-  # Sidekiq.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.add MyServerHook
-  #     chain.remove ActiveRecord
+  #   Sidekiq.configure_server do |config|
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #       chain.remove ActiveRecord
+  #     end
   #   end
-  # end
   #
   # To insert immediately preceding another entry:
   #
-  # Sidekiq.configure_client do |config|
-  #   config.client_middleware do |chain|
-  #     chain.insert_before ActiveRecord, MyClientHook
+  #   Sidekiq.configure_client do |config|
+  #     config.client_middleware do |chain|
+  #       chain.insert_before ActiveRecord, MyClientHook
+  #     end
   #   end
-  # end
   #
   # To insert immediately after another entry:
   #
-  # Sidekiq.configure_client do |config|
-  #   config.client_middleware do |chain|
-  #     chain.insert_after ActiveRecord, MyClientHook
+  #   Sidekiq.configure_client do |config|
+  #     config.client_middleware do |chain|
+  #       chain.insert_after ActiveRecord, MyClientHook
+  #     end
   #   end
-  # end
   #
   # This is an example of a minimal server middleware:
   #
-  # class MyServerHook
-  #   include Sidekiq::ServerMiddleware
-  #   def call(job_instance, msg, queue)
-  #     logger.info "Before job"
-  #     redis {|conn| conn.get("foo") } # do something in Redis
-  #     yield
-  #     logger.info "After job"
+  #   class MyServerHook
+  #     include Sidekiq::ServerMiddleware
+  #
+  #     def call(job_instance, msg, queue)
+  #       logger.info "Before job"
+  #       redis {|conn| conn.get("foo") } # do something in Redis
+  #       yield
+  #       logger.info "After job"
+  #     end
   #   end
-  # end
   #
   # This is an example of a minimal client middleware, note
   # the method must return the result or the job will not push
   # to Redis:
   #
-  # class MyClientHook
-  #   include Sidekiq::ClientMiddleware
-  #   def call(job_class, msg, queue, redis_pool)
-  #     logger.info "Before push"
-  #     result = yield
-  #     logger.info "After push"
-  #     result
+  #   class MyClientHook
+  #     include Sidekiq::ClientMiddleware
+  #
+  #     def call(job_class, msg, queue, redis_pool)
+  #       logger.info "Before push"
+  #       result = yield
+  #       logger.info "After push"
+  #       result
+  #     end
   #   end
-  # end
   #
   module Middleware
     class Chain
       include Enumerable
 
+      # A unique instance of the middleware chain is created for
+      # each job executed in order to be thread-safe.
+      # @param copy [Sidekiq::Middleware::Chain] New instance of Chain
+      # @returns nil
       def initialize_copy(copy)
         copy.instance_variable_set(:@entries, entries.dup)
+        nil
       end
 
+      # Iterate through each middleware in the chain
       def each(&block)
         entries.each(&block)
       end
 
-      def initialize(config = nil)
+      # @api private
+      def initialize(config = nil) # :nodoc:
         @config = config
         @entries = nil
         yield self if block_given?
@@ -91,20 +105,33 @@ module Sidekiq
         @entries ||= []
       end
 
+      # Remove all middleware matching the given Class
+      # @param klass [Class]
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
 
+      # Add the given middleware to the end of the chain.
+      # Sidekiq will call `klass.new(*args)` to create a clean
+      # copy of your middleware for every job executed.
+      #
+      #   chain.add(Statsd::Metrics, { collector: "localhost:8125" })
+      #
+      # @param klass [Class] Your middleware class
+      # @param *args [Array<Object>] Set of arguments to pass to every instance of your middleware
       def add(klass, *args)
         remove(klass)
         entries << Entry.new(@config, klass, *args)
       end
 
+      # Identical to {#add} except the middleware is added to the front of the chain.
       def prepend(klass, *args)
         remove(klass)
         entries.insert(0, Entry.new(@config, klass, *args))
       end
 
+      # Inserts +newklass+ before +oldklass+ in the chain.
+      # Useful if one middleware must run before another middleware.
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
@@ -112,6 +139,8 @@ module Sidekiq
         entries.insert(i, new_entry)
       end
 
+      # Inserts +newklass+ after +oldklass+ in the chain.
+      # Useful if one middleware must run after another middleware.
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
@@ -119,10 +148,12 @@ module Sidekiq
         entries.insert(i + 1, new_entry)
       end
 
+      # @return [Boolean] if the given class is already in the chain
       def exists?(klass)
         any? { |entry| entry.klass == klass }
       end
 
+      # @return [Boolean] if the chain contains no middleware
       def empty?
         @entries.nil? || @entries.empty?
       end
@@ -135,6 +166,8 @@ module Sidekiq
         entries.clear
       end
 
+      # Used by Sidekiq to execute the middleware at runtime
+      # @api private
       def invoke(*args)
         return yield if empty?
 
@@ -152,6 +185,8 @@ module Sidekiq
 
     private
 
+    # Represents each link in the middleware chain
+    # @api private
     class Entry
       attr_reader :klass
 

data/lib/sidekiq/middleware/current_attributes.rb

@@ -23,10 +23,12 @@ module Sidekiq
 
       def call(_, job, _, _)
         attrs = @klass.attributes
-        if job.has_key?("cattr")
-          job["cattr"].merge!(attrs)
-        else
-          job["cattr"] = attrs
+        if attrs.any?
+          if job.has_key?("cattr")
+            job["cattr"].merge!(attrs)
+          else
+            job["cattr"] = attrs
+          end
         end
         yield
       end

data/lib/sidekiq/processor.rb

@@ -152,8 +152,14 @@ module Sidekiq
         job_hash = Sidekiq.load_json(jobstr)
       rescue => ex
         handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
-        # we can't notify because the job isn't a valid hash payload.
-        DeadSet.new.kill(jobstr, notify_failure: false)
+        now = Time.now.to_f
+        config.redis do |conn|
+          conn.multi do |xa|
+            xa.zadd("dead", now.to_s, jobstr)
+            xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
+            xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+          end
+        end
         return uow.acknowledge
       end
 
@@ -174,7 +180,7 @@ module Sidekiq
         # signals that we created a retry successfully.  We can acknowlege the job.
         ack = true
         e = h.cause || h
-        handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
+        handle_exception(e, {context: "Job raised exception", job: job_hash})
         raise e
       rescue Exception => ex
         # Unexpected error!  This is very bad and indicates an exception that got past

data/lib/sidekiq/redis_connection.rb

@@ -46,8 +46,6 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
-
         # Issue #3303, redis-rb will silently retry an operation.
         # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
         # is performed twice but I believe this is much, much rarer

data/lib/sidekiq/scheduled.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "sidekiq"
-require "sidekiq/api"
 require "sidekiq/component"
 
 module Sidekiq
@@ -183,13 +182,8 @@ module Sidekiq
       end
 
       def process_count
-        # The work buried within Sidekiq::ProcessSet#cleanup can be
-        # expensive at scale. Cut it down by 90% with this counter.
-        # NB: This method is only called by the scheduler thread so we
-        # don't need to worry about the thread safety of +=.
-        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
+        pcount = Sidekiq.redis { |conn| conn.scard("processes") }
         pcount = 1 if pcount == 0
-        @count_calls += 1
         pcount
       end
 

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.5.1"
+  VERSION = "6.5.6"
 end

data/lib/sidekiq/web/action.rb

@@ -15,11 +15,11 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
+      throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
-      throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
+      throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
     end
 
     def params
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
+      [200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -60,6 +60,19 @@ module Sidekiq
       erb(:dashboard)
     end
 
+    get "/metrics" do
+      q = Sidekiq::Metrics::Query.new
+      @query_result = q.top_jobs
+      erb(:metrics)
+    end
+
+    get "/metrics/:name" do
+      @name = route_params[:name]
+      q = Sidekiq::Metrics::Query.new
+      @query_result = q.for_job(@name)
+      erb(:metrics_for_job)
+    end
+
     get "/busy" do
       erb(:busy)
     end
@@ -299,7 +312,7 @@ module Sidekiq
 
     def call(env)
       action = self.class.match(env)
-      return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
+      return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
 
       app = @klass
       resp = catch(:halt) do
@@ -316,10 +329,10 @@ module Sidekiq
       else
         # rendered content goes here
         headers = {
-          "Content-Type" => "text/html",
-          "Cache-Control" => "private, no-store",
-          "Content-Language" => action.locale,
-          "Content-Security-Policy" => CSP_HEADER
+          "content-type" => "text/html",
+          "cache-control" => "private, no-store",
+          "content-language" => action.locale,
+          "content-security-policy" => CSP_HEADER
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]

data/lib/sidekiq/web/helpers.rb

@@ -15,7 +15,7 @@ module Sidekiq
       # so extensions can be localized
       @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.load(File.open(file))
+          strs = YAML.safe_load(File.open(file))
           global.merge!(strs[lang])
         end
       end
@@ -148,6 +148,29 @@ module Sidekiq
       @processes ||= Sidekiq::ProcessSet.new
     end
 
+    # Sorts processes by hostname following the natural sort order so that
+    # 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
+    # '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
+    def sorted_processes
+      @sorted_processes ||= begin
+        return processes unless processes.all? { |p| p["hostname"] }
+
+        split_characters = /[._-]/
+
+        padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
+
+        processes.to_a.sort_by do |process|
+          process["hostname"].split(split_characters).map do |substring|
+            # Left-pad the substring with '0' if it starts with a number or 'a'
+            # otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
+            padding_char = substring[0].match?(/\d/) ? "0" : "a"
+
+            substring.rjust(padding, padding_char)
+          end
+        end
+      end
+    end
+
     def stats
       @stats ||= Sidekiq::Stats.new
     end

data/lib/sidekiq/web.rb

@@ -33,6 +33,10 @@ module Sidekiq
       "Dead" => "morgue"
     }
 
+    if ENV["SIDEKIQ_METRICS_BETA"] == "1"
+      DEFAULT_TABS["Metrics"] = "metrics"
+    end
+
     class << self
       def settings
         self
@@ -144,7 +148,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],

data/lib/sidekiq.rb

@@ -34,7 +34,10 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
-      heartbeat: []
+      # triggers when we fire the first heartbeat on startup OR repairing a network partition
+      heartbeat: [],
+      # triggers on EVERY heartbeat call, every 10 seconds
+      beat: []
     },
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
@@ -84,6 +87,11 @@ module Sidekiq
     logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
   end
 
+  # DEFAULT_ERROR_HANDLER is a constant that allows the default error handler to
+  # be referenced. It must be defined here, after the default_error_handler
+  # method is defined.
+  DEFAULT_ERROR_HANDLER = method(:default_error_handler)
+
   @config = DEFAULTS.dup
   def self.options
     logger.warn "`config.options[:key] = value` is deprecated, use `config[:key] = value`: #{caller(1..2)}"

data/sidekiq.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
     "source_code_uri" => "https://github.com/mperham/sidekiq"
   }
 
-  gem.add_dependency "redis", ">= 4.2.0"
-  gem.add_dependency "connection_pool", ">= 2.2.2"
+  gem.add_dependency "redis", "<5", ">= 4.5.0"
+  gem.add_dependency "connection_pool", ">= 2.2.5"
   gem.add_dependency "rack", "~> 2.0"
 end

data/web/assets/javascripts/application.js

@@ -63,7 +63,7 @@ function addPollingListeners(_event)  {
 function addDataToggleListeners(event) {
   var source = event.target || event.srcElement;
   var targName = source.getAttribute("data-toggle");
-  var full = document.getElementById(targName + "_full");
+  var full = document.getElementById(targName);
   if (full.style.display == "block") {
     full.style.display = 'none';
   } else {