sidekiq 6.4.1 → 6.5.7

data/lib/sidekiq/ring_buffer.rb

@@ -0,0 +1,29 @@
+require "forwardable"
+
+module Sidekiq
+  class RingBuffer
+    include Enumerable
+    extend Forwardable
+    def_delegators :@buf, :[], :each, :size
+
+    def initialize(size, default = 0)
+      @size = size
+      @buf = Array.new(size, default)
+      @index = 0
+    end
+
+    def <<(element)
+      @buf[@index % @size] = element
+      @index += 1
+      element
+    end
+
+    def buffer
+      @buf
+    end
+
+    def reset(default = 0)
+      @buf.fill(default)
+    end
+  end
+end

data/lib/sidekiq/scheduled.rb

@@ -1,8 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq"
-require "sidekiq/util"
-require "sidekiq/api"
+require "sidekiq/component"
 
 module Sidekiq
   module Scheduled
@@ -52,8 +51,8 @@ module Sidekiq
           @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
-        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
-      rescue Redis::CommandError => e
+        conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
+      rescue RedisConnection.adapter::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
         @lua_zpopbyscore_sha = nil
@@ -67,12 +66,13 @@ module Sidekiq
     # just pops the job back onto its original queue so the
     # workers can pick it up like any other job.
     class Poller
-      include Util
+      include Sidekiq::Component
 
       INITIAL_WAIT = 10
 
-      def initialize
-        @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+      def initialize(options)
+        @config = options
+        @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
@@ -100,7 +100,7 @@ module Sidekiq
             enqueue
             wait
           end
-          Sidekiq.logger.info("Scheduler exiting...")
+          logger.info("Scheduler exiting...")
         }
       end
 
@@ -147,13 +147,16 @@ module Sidekiq
         # As we run more processes, the scheduling interval average will approach an even spread
         # between 0 and poll interval so we don't need this artifical boost.
         #
-        if process_count < 10
+        count = process_count
+        interval = poll_interval_average(count)
+
+        if count < 10
           # For small clusters, calculate a random interval that is ±50% the desired average.
-          poll_interval_average * rand + poll_interval_average.to_f / 2
+          interval * rand + interval.to_f / 2
         else
           # With 10+ processes, we should have enough randomness to get decent polling
           # across the entire timespan
-          poll_interval_average * rand
+          interval * rand
         end
       end
 
@@ -170,38 +173,64 @@ module Sidekiq
       # the same time: the thundering herd problem.
       #
       # We only do this if poll_interval_average is unset (the default).
-      def poll_interval_average
-        Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
+      def poll_interval_average(count)
+        @config[:poll_interval_average] || scaled_poll_interval(count)
       end
 
       # Calculates an average poll interval based on the number of known Sidekiq processes.
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
-      def scaled_poll_interval
-        process_count * Sidekiq.options[:average_scheduled_poll_interval]
+      def scaled_poll_interval(process_count)
+        process_count * @config[:average_scheduled_poll_interval]
       end
 
       def process_count
-        # The work buried within Sidekiq::ProcessSet#cleanup can be
-        # expensive at scale. Cut it down by 90% with this counter.
-        # NB: This method is only called by the scheduler thread so we
-        # don't need to worry about the thread safety of +=.
-        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
+        pcount = Sidekiq.redis { |conn| conn.scard("processes") }
         pcount = 1 if pcount == 0
-        @count_calls += 1
         pcount
       end
 
+      # A copy of Sidekiq::ProcessSet#cleanup because server
+      # should never depend on sidekiq/api.
+      def cleanup
+        # dont run cleanup more than once per minute
+        return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+
+        count = 0
+        Sidekiq.redis do |conn|
+          procs = conn.sscan_each("processes").to_a
+          heartbeats = conn.pipelined { |pipeline|
+            procs.each do |key|
+              pipeline.hget(key, "info")
+            end
+          }
+
+          # the hash named key has an expiry of 60 seconds.
+          # if it's not found, that means the process has not reported
+          # in to Redis and probably died.
+          to_prune = procs.select.with_index { |proc, i|
+            heartbeats[i].nil?
+          }
+          count = conn.srem("processes", to_prune) unless to_prune.empty?
+        end
+        count
+      end
+
       def initial_wait
-        # Have all processes sleep between 5-15 seconds.  10 seconds
-        # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+        # Have all processes sleep between 5-15 seconds. 10 seconds to give time for
+        # the heartbeat to register (if the poll interval is going to be calculated by the number
         # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
         total = 0
-        total += INITIAL_WAIT unless Sidekiq.options[:poll_interval_average]
+        total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
         @sleeper.pop(total)
       rescue Timeout::Error
+      ensure
+        # periodically clean out the `processes` set in Redis which can collect
+        # references to dead processes over time. The process count affects how
+        # often we scan for scheduled jobs.
+        cleanup
       end
     end
   end

data/lib/sidekiq/testing/inline.rb

@@ -4,7 +4,7 @@ require "sidekiq/testing"
 
 ##
 # The Sidekiq inline infrastructure overrides perform_async so that it
-# actually calls perform instead. This allows workers to be run inline in a
+# actually calls perform instead. This allows jobs to be run inline in a
 # testing environment.
 #
 # This is similar to `Resque.inline = true` functionality.
@@ -15,8 +15,8 @@ require "sidekiq/testing"
 #
 #   $external_variable = 0
 #
-#   class ExternalWorker
-#     include Sidekiq::Worker
+#   class ExternalJob
+#     include Sidekiq::Job
 #
 #     def perform
 #       $external_variable = 1
@@ -24,7 +24,7 @@ require "sidekiq/testing"
 #   end
 #
 #   assert_equal 0, $external_variable
-#   ExternalWorker.perform_async
+#   ExternalJob.perform_async
 #   assert_equal 1, $external_variable
 #
 Sidekiq::Testing.inline!

data/lib/sidekiq/testing.rb

@@ -101,20 +101,20 @@ module Sidekiq
     ##
     # The Queues class is only for testing the fake queue implementation.
     # There are 2 data structures involved in tandem. This is due to the
-    # Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
+    # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
     # to the array. Because the array was dervied from a filter of the total
     # jobs enqueued, it appeared as though the array didn't change.
     #
     # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
-    # on the queue, and another with keys of the worker names, so the array for
-    # QueueWorker.jobs is a straight reference to a real array.
+    # on the queue, and another with keys of the job type, so the array for
+    # HardJob.jobs is a straight reference to a real array.
     #
     # Queue-based hash:
     #
     # {
     #   "default"=>[
     #     {
-    #       "class"=>"TestTesting::QueueWorker",
+    #       "class"=>"TestTesting::HardJob",
     #       "args"=>[1, 2],
     #       "retry"=>true,
     #       "queue"=>"default",
@@ -124,12 +124,12 @@ module Sidekiq
     #   ]
     # }
     #
-    # Worker-based hash:
+    # Job-based hash:
     #
     # {
-    #   "TestTesting::QueueWorker"=>[
+    #   "TestTesting::HardJob"=>[
     #     {
-    #       "class"=>"TestTesting::QueueWorker",
+    #       "class"=>"TestTesting::HardJob",
     #       "args"=>[1, 2],
     #       "retry"=>true,
     #       "queue"=>"default",
@@ -144,14 +144,14 @@ module Sidekiq
     #   require 'sidekiq/testing'
     #
     #   assert_equal 0, Sidekiq::Queues["default"].size
-    #   HardWorker.perform_async(:something)
+    #   HardJob.perform_async(:something)
     #   assert_equal 1, Sidekiq::Queues["default"].size
     #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
     #
-    # You can also clear all workers' jobs:
+    # You can also clear all jobs:
     #
     #   assert_equal 0, Sidekiq::Queues["default"].size
-    #   HardWorker.perform_async(:something)
+    #   HardJob.perform_async(:something)
     #   Sidekiq::Queues.clear_all
     #   assert_equal 0, Sidekiq::Queues["default"].size
     #
@@ -170,35 +170,36 @@ module Sidekiq
 
       def push(queue, klass, job)
         jobs_by_queue[queue] << job
-        jobs_by_worker[klass] << job
+        jobs_by_class[klass] << job
       end
 
       def jobs_by_queue
         @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
       end
 
-      def jobs_by_worker
-        @jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
+      def jobs_by_class
+        @jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
       end
+      alias_method :jobs_by_worker, :jobs_by_class
 
       def delete_for(jid, queue, klass)
         jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
-        jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
+        jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
       end
 
       def clear_for(queue, klass)
-        jobs_by_queue[queue].clear
-        jobs_by_worker[klass].clear
+        jobs_by_queue[queue.to_s].clear
+        jobs_by_class[klass].clear
       end
 
       def clear_all
         jobs_by_queue.clear
-        jobs_by_worker.clear
+        jobs_by_class.clear
       end
     end
   end
 
-  module Worker
+  module Job
     ##
     # The Sidekiq testing infrastructure overrides perform_async
     # so that it does not actually touch the network.  Instead it
@@ -212,16 +213,16 @@ module Sidekiq
     #
     #   require 'sidekiq/testing'
     #
-    #   assert_equal 0, HardWorker.jobs.size
-    #   HardWorker.perform_async(:something)
-    #   assert_equal 1, HardWorker.jobs.size
-    #   assert_equal :something, HardWorker.jobs[0]['args'][0]
+    #   assert_equal 0, HardJob.jobs.size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, HardJob.jobs.size
+    #   assert_equal :something, HardJob.jobs[0]['args'][0]
     #
     #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
     #   MyMailer.delay.send_welcome_email('foo@example.com')
     #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
     #
-    # You can also clear and drain all workers' jobs:
+    # You can also clear and drain all job types:
     #
     #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
     #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
@@ -241,14 +242,14 @@ module Sidekiq
     #
     #   RSpec.configure do |config|
     #     config.before(:each) do
-    #       Sidekiq::Worker.clear_all
+    #       Sidekiq::Job.clear_all
     #     end
     #   end
     #
     # or for acceptance testing, i.e. with cucumber:
     #
     #   AfterStep do
-    #     Sidekiq::Worker.drain_all
+    #     Sidekiq::Job.drain_all
     #   end
     #
     #   When I sign up as "foo@example.com"
@@ -262,7 +263,7 @@ module Sidekiq
 
       # Jobs queued for this worker
       def jobs
-        Queues.jobs_by_worker[to_s]
+        Queues.jobs_by_class[to_s]
       end
 
       # Clear all jobs for this worker
@@ -288,11 +289,11 @@ module Sidekiq
       end
 
       def process_job(job)
-        worker = new
-        worker.jid = job["jid"]
-        worker.bid = job["bid"] if worker.respond_to?(:bid=)
-        Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
-          execute_job(worker, job["args"])
+        inst = new
+        inst.jid = job["jid"]
+        inst.bid = job["bid"] if inst.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
+          execute_job(inst, job["args"])
         end
       end
 
@@ -306,18 +307,18 @@ module Sidekiq
         Queues.jobs_by_queue.values.flatten
       end
 
-      # Clear all queued jobs across all workers
+      # Clear all queued jobs
       def clear_all
         Queues.clear_all
       end
 
-      # Drain all queued jobs across all workers
+      # Drain (execute) all queued jobs
       def drain_all
         while jobs.any?
-          worker_classes = jobs.map { |job| job["class"] }.uniq
+          job_classes = jobs.map { |job| job["class"] }.uniq
 
-          worker_classes.each do |worker_class|
-            Sidekiq::Testing.constantize(worker_class).drain
+          job_classes.each do |job_class|
+            Sidekiq::Testing.constantize(job_class).drain
           end
         end
       end

data/lib/sidekiq/transaction_aware_client.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq/client"
+
+module Sidekiq
+  class TransactionAwareClient
+    def initialize(redis_pool)
+      @redis_client = Client.new(redis_pool)
+    end
+
+    def push(item)
+      # pre-allocate the JID so we can return it immediately and
+      # save it to the database as part of the transaction.
+      item["jid"] ||= SecureRandom.hex(12)
+      AfterCommitEverywhere.after_commit { @redis_client.push(item) }
+      item["jid"]
+    end
+
+    ##
+    # We don't provide transactionality for push_bulk because we don't want
+    # to hold potentially hundreds of thousands of job records in memory due to
+    # a long running enqueue process.
+    def push_bulk(items)
+      @redis_client.push_bulk(items)
+    end
+  end
+end
+
+##
+# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
+module Sidekiq
+  def self.transactional_push!
+    begin
+      require "after_commit_everywhere"
+    rescue LoadError
+      Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
+      raise
+    end
+
+    default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
+    true
+  end
+end

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.4.1"
+  VERSION = "6.5.7"
 end

data/lib/sidekiq/web/action.rb

@@ -15,11 +15,11 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
+      throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
-      throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
+      throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
     end
 
     def params
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
+      [200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -60,6 +60,19 @@ module Sidekiq
       erb(:dashboard)
     end
 
+    get "/metrics" do
+      q = Sidekiq::Metrics::Query.new
+      @query_result = q.top_jobs
+      erb(:metrics)
+    end
+
+    get "/metrics/:name" do
+      @name = route_params[:name]
+      q = Sidekiq::Metrics::Query.new
+      @query_result = q.for_job(@name)
+      erb(:metrics_for_job)
+    end
+
     get "/busy" do
       erb(:busy)
     end
@@ -299,7 +312,7 @@ module Sidekiq
 
     def call(env)
       action = self.class.match(env)
-      return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
+      return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
 
       app = @klass
       resp = catch(:halt) do
@@ -316,10 +329,10 @@ module Sidekiq
       else
         # rendered content goes here
         headers = {
-          "Content-Type" => "text/html",
-          "Cache-Control" => "private, no-store",
-          "Content-Language" => action.locale,
-          "Content-Security-Policy" => CSP_HEADER
+          "content-type" => "text/html",
+          "cache-control" => "private, no-store",
+          "content-language" => action.locale,
+          "content-security-policy" => CSP_HEADER
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]

data/lib/sidekiq/web/csrf_protection.rb

@@ -143,7 +143,7 @@ module Sidekiq
         one_time_pad = SecureRandom.random_bytes(token.length)
         encrypted_token = xor_byte_strings(one_time_pad, token)
         masked_token = one_time_pad + encrypted_token
-        Base64.strict_encode64(masked_token)
+        Base64.urlsafe_encode64(masked_token)
       end
 
       # Essentially the inverse of +mask_token+.
@@ -169,7 +169,7 @@ module Sidekiq
       end
 
       def decode_token(token)
-        Base64.strict_decode64(token)
+        Base64.urlsafe_decode64(token)
       end
 
       def xor_byte_strings(s1, s2)

data/lib/sidekiq/web/helpers.rb

@@ -15,7 +15,7 @@ module Sidekiq
       # so extensions can be localized
       @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.load(File.open(file))
+          strs = YAML.safe_load(File.open(file))
           global.merge!(strs[lang])
         end
       end
@@ -140,14 +140,37 @@ module Sidekiq
       params[:direction] == "asc" ? "↑" : "↓"
     end
 
-    def workers
-      @workers ||= Sidekiq::Workers.new
+    def workset
+      @work ||= Sidekiq::WorkSet.new
     end
 
     def processes
       @processes ||= Sidekiq::ProcessSet.new
     end
 
+    # Sorts processes by hostname following the natural sort order so that
+    # 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
+    # '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
+    def sorted_processes
+      @sorted_processes ||= begin
+        return processes unless processes.all? { |p| p["hostname"] }
+
+        split_characters = /[._-]+/
+
+        padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
+
+        processes.to_a.sort_by do |process|
+          process["hostname"].split(split_characters).map do |substring|
+            # Left-pad the substring with '0' if it starts with a number or 'a'
+            # otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
+            padding_char = substring[0].match?(/\d/) ? "0" : "a"
+
+            substring.rjust(padding, padding_char)
+          end
+        end
+      end
+    end
+
     def stats
       @stats ||= Sidekiq::Stats.new
     end
@@ -175,7 +198,7 @@ module Sidekiq
     end
 
     def current_status
-      workers.size == 0 ? "idle" : "active"
+      workset.size == 0 ? "idle" : "active"
     end
 
     def relative_time(time)
@@ -301,7 +324,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end

data/lib/sidekiq/web.rb

@@ -33,6 +33,10 @@ module Sidekiq
       "Dead" => "morgue"
     }
 
+    if ENV["SIDEKIQ_METRICS_BETA"] == "1"
+      DEFAULT_TABS["Metrics"] = "metrics"
+    end
+
     class << self
       def settings
         self
@@ -144,7 +148,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],