sidekiq 8.1.0 → 8.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63658318932ac6b7045211590b07f84ed9e1e6398f908ace790a0df20d5ac1f0
-  data.tar.gz: 0c3f5e69ada7529bdac7392acb1c56ef2a19b8ed13210be8c36aac19858c6df4
+  metadata.gz: 0d21db4cf06c0b4d0b5fb770162e289403a76ed996d6c99613cf45083f0e671d
+  data.tar.gz: 07cf848cd5de112f3153ecd0014991cbf2abf9e66bc0847d2b7845ec7a726329
 SHA512:
-  metadata.gz: 9a5e95a2faccbbf42f8e651c4882c9495ad72b0500bb8fe77fa2a93fb4aa6f8cb96fc820621cd1910fb4b00accf35d75a99d548b3cc8ccdd41a18b5620a9ad73
-  data.tar.gz: 1bc66490ee07c548c71f680fb280e876efa7eeb8c6f234277df73211ce05c5ae29f5c9bc13064972da945568d1df6634252a6c22a7c5298bdd436b7266246c53
+  metadata.gz: 395d346f8b5227480e4d830b3c1f219a35bb0e2527283f9c00a6a4043c1ebe271998874ae53240b6e7dabca89ce8ad939d0a3969da2b33c3e4155fccbebae9ba
+  data.tar.gz: 20f44288fd8990544569af7f2a87d90d4ce898b2ce760cc95dd2f30c5a0c4ffc90bcc087cbfa22dbc188a904cc2e817756a83b5b3ffd648f75dc0413d3bec3ff

data/Changes.md

@@ -2,6 +2,35 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.2
+----------
+
+- Initial release for `kiq`, Sidekiq's official terminal UI:
+```
+bundle exec kiq
+```
+Use REDIS_URL or REDIS_PROVIDER to point `kiq` to Redis.
+- Mutation during iteration in `SortedSet#each` caused it to miss half of the jobs [#6936]
+- Fix edge case resulting in nil crash on /busy page [#6954]
+
+8.1.1
+----------
+
+- **DEPRECATION** `require 'sidekiq/testing'` and
+  `require 'sidekiq/testing/inline'`.
+  Add new `Sidekiq.testing!(mode)` API [#6931]
+  Requiring code should not enable process-wide changes.
+```ruby
+# Old, implicit
+require "sidekiq/testing"
+require "sidekiq/testing/inline"
+# New, more explicit
+Sidekiq.testing!(:fake)
+Sidekiq.testing!(:inline)
+```
+- Fix race condition with Stop button in UI [#6935]
+- Fix javascript error handler [#6893]
+
 8.1.0
 ----------
 

data/README.md

@@ -90,7 +90,7 @@ Useful resources:
 * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
 
 Every Thursday morning is Sidekiq Office Hour: I video chat and answer questions.
-See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
+See the [Sidekiq support page](https://sidekiq.org/support/) for details.
 
 Contributing
 -----------------

data/bin/kiq

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+# This requires the default gemset so Sidekiq Pro
+# and Sidekiq Enterprise can load any code extensions.
+Bundler.require(:default, :tui)
+
+require_relative "../lib/sidekiq/tui"
+
+# Run any load hooks registered during Bundler.require
+Sidekiq.loader.run_load_hooks(:tui)
+
+tt = Sidekiq::TUI.new(Sidekiq.default_configuration)
+
+RatatuiRuby.run do |tui|
+  tt.prepare(tui)
+  tt.run_loop
+end

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -61,18 +61,21 @@ begin
 
         # @api private
         def enqueue(job)
-          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
-            wrapped: job.class,
-            queue: job.queue_name
-          ).perform_async(job.serialize)
+          # NB: Active Job only serializes keys it recognizes. We
+          # cannot set arbitrary key/values here.
+          options = {wrapped: job.class, queue: job.queue_name}
+          options[:profile] = job.profile if job.respond_to?(:profile) && !job.profile.nil?
+
+          wrapper = Sidekiq::ActiveJob::Wrapper.set(options)
+          job.provider_job_id = wrapper.perform_async(job.serialize)
         end
 
         # @api private
         def enqueue_at(job, timestamp)
-          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
-            wrapped: job.class,
-            queue: job.queue_name
-          ).perform_at(timestamp, job.serialize)
+          options = {wrapped: job.class, queue: job.queue_name}
+          options[:profile] = job.profile if job.respond_to?(:profile) && !job.profile.nil?
+
+          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(options).perform_at(timestamp, job.serialize)
         end
 
         # @api private

data/lib/generators/sidekiq/job_generator.rb

@@ -14,7 +14,10 @@ module Sidekiq
       end
 
       def create_job_file
-        template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
+        template(
+          "job.rb.erb",
+          File.join(jobs_directory, class_path, "#{file_name}_job.rb")
+        )
       end
 
       def create_test_file
@@ -31,7 +34,8 @@ module Sidekiq
 
       def create_job_spec
         template_file = File.join(
-          "spec/sidekiq",
+          "spec",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_spec.rb"
         )
@@ -40,7 +44,8 @@ module Sidekiq
 
       def create_job_test
         template_file = File.join(
-          "test/sidekiq",
+          "test",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_test.rb"
         )
@@ -54,6 +59,13 @@ module Sidekiq
       def test_framework
         ::Rails.application.config.generators.options[:rails][:test_framework]
       end
+
+      # Can be set in an initializer or in application configuration
+      # with Rails.application.config.generators.options[:sidekiq][:jobs_directory] = "app/jobs"
+      # to change the directory that the job files are generated in to.
+      def jobs_directory
+        ::Rails.application.config.generators.options[:sidekiq].fetch(:jobs_directory, "app/sidekiq")
+      end
     end
   end
 end

data/lib/sidekiq/api.rb

@@ -17,12 +17,37 @@ require "sidekiq/metrics/query"
 #
 
 module Sidekiq
+  module ApiUtils
+    # @api private
+    # Calculate the latency in seconds for a job based on its enqueued timestamp
+    # @param job [Hash] the job hash
+    # @return [Float] latency in seconds
+    def calculate_latency(job)
+      timestamp = job["enqueued_at"] || job["created_at"]
+      return 0.0 unless timestamp
+
+      if timestamp.is_a?(Float)
+        # old format
+        Time.now.to_f - timestamp
+      else
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+        (now - timestamp) / 1000.0
+      end
+    end
+  end
+
   # Retrieve runtime statistics from Redis regarding
   # this Sidekiq cluster.
   #
   #   stat = Sidekiq::Stats.new
   #   stat.processed
   class Stats
+    QueueSummary = Data.define(:name, :size, :latency, :paused) do
+      alias_method :paused?, :paused
+    end
+
+    include ApiUtils
+
     def initialize
       fetch_stats_fast!
     end
@@ -63,6 +88,7 @@ module Sidekiq
       stat :default_queue_latency
     end
 
+    # @return [Hash{String => Integer}] a hash of queue names to their lengths
     def queues
       Sidekiq.redis do |conn|
         queues = conn.sscan("queues").to_a
@@ -78,6 +104,41 @@ module Sidekiq
       end
     end
 
+    # More detailed information about each queue: name, size, latency, paused status
+    # @return [Array<QueueSummary>]
+    def queue_summaries
+      Sidekiq.redis do |conn|
+        queues = conn.sscan("queues").to_a
+        return [] if queues.empty?
+
+        results = conn.pipelined { |pipeline|
+          queues.each do |queue|
+            pipeline.llen("queue:#{queue}")
+            pipeline.lindex("queue:#{queue}", -1)
+            pipeline.sismember("paused", queue)
+          end
+        }
+
+        queue_summaries = []
+        queues.each_with_index do |name, idx|
+          size = results[idx * 3]
+          last_item = results[idx * 3 + 1]
+          paused = results[idx * 3 + 2] > 0
+
+          latency = if last_item
+            job = Sidekiq.load_json(last_item)
+            calculate_latency(job)
+          else
+            0.0
+          end
+
+          queue_summaries << QueueSummary.new(name:, size:, latency:, paused:)
+        end
+
+        queue_summaries.sort_by { |qd| -qd.size }
+      end
+    end
+
     # O(1) redis calls
     # @api private
     def fetch_stats_fast!
@@ -100,19 +161,7 @@ module Sidekiq
           {}
         end
 
-        enqueued_at = job["enqueued_at"]
-        if enqueued_at
-          if enqueued_at.is_a?(Float)
-            # old format
-            now = Time.now.to_f
-            now - enqueued_at
-          else
-            now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-            (now - enqueued_at) / 1000.0
-          end
-        else
-          0.0
-        end
+        calculate_latency(job)
       else
         0.0
       end
@@ -235,6 +284,7 @@ module Sidekiq
   #   end
   class Queue
     include Enumerable
+    include ApiUtils
 
     ##
     # Fetch all known queues within Redis.
@@ -245,6 +295,7 @@ module Sidekiq
     end
 
     attr_reader :name
+    alias_method :id, :name
 
     # @param name [String] the name of the queue
     def initialize(name = "default")
@@ -277,19 +328,7 @@ module Sidekiq
       return 0.0 unless entry
 
       job = Sidekiq.load_json(entry)
-      enqueued_at = job["enqueued_at"]
-      if enqueued_at
-        if enqueued_at.is_a?(Float)
-          # old format
-          now = Time.now.to_f
-          now - enqueued_at
-        else
-          now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-          (now - enqueued_at) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(job)
     end
 
     def each
@@ -352,6 +391,8 @@ module Sidekiq
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
   class JobRecord
+    include ApiUtils
+
     # the parsed Hash of job data
     # @!attribute [r] Item
     attr_reader :item
@@ -478,17 +519,7 @@ module Sidekiq
     end
 
     def latency
-      timestamp = @item["enqueued_at"] || @item["created_at"]
-      if timestamp
-        if timestamp.is_a?(Float)
-          # old format
-          Time.now.to_f - timestamp
-        else
-          (::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) - timestamp) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(@item)
     end
 
     # Remove this job from the queue
@@ -553,17 +584,24 @@ module Sidekiq
   # could be the scheduled time for it to run (e.g. scheduled set),
   # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
-    attr_reader :score
     attr_reader :parent
 
     # :nodoc:
     # @api private
     def initialize(parent, score, item)
       super(item)
-      @score = Float(score)
+      @score = score
       @parent = parent
     end
 
+    def score
+      Float(@score)
+    end
+
+    def id
+      "#{@score}|#{item["jid"]}"
+    end
+
     # The timestamp associated with this entry
     def at
       Time.at(score).utc
@@ -574,7 +612,7 @@ module Sidekiq
       if @value
         @parent.delete_by_value(@parent.name, @value)
       else
-        @parent.delete_by_jid(score, jid)
+        @parent.delete_by_jid(@score, jid)
       end
     end
 
@@ -583,7 +621,7 @@ module Sidekiq
     # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
-        conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
+        conn.zincrby(@parent.name, at.to_f - score, Sidekiq.dump_json(@item))
       end
     end
 
@@ -619,38 +657,8 @@ module Sidekiq
 
     private
 
-    def remove_job
-      Sidekiq.redis do |conn|
-        results = conn.multi { |transaction|
-          transaction.zrange(parent.name, score, score, "BYSCORE")
-          transaction.zremrangebyscore(parent.name, score, score)
-        }.first
-
-        if results.size == 1
-          yield results.first
-        else
-          # multiple jobs with the same score
-          # find the one with the right JID and push it
-          matched, nonmatched = results.partition { |message|
-            if message.index(jid)
-              msg = Sidekiq.load_json(message)
-              msg["jid"] == jid
-            else
-              false
-            end
-          }
-
-          msg = matched.first
-          yield msg if msg
-
-          # push the rest back onto the sorted set
-          conn.multi do |transaction|
-            nonmatched.each do |message|
-              transaction.zadd(parent.name, score.to_f.to_s, message)
-            end
-          end
-        end
-      end
+    def remove_job(&)
+      parent.remove_job(self, &)
     end
   end
 
@@ -819,6 +827,46 @@ module Sidekiq
       nil
     end
 
+    def remove_job(entry)
+      score = entry.score
+      jid = entry.jid
+      Sidekiq.redis do |conn|
+        results = conn.multi { |transaction|
+          transaction.zrange(name, score, score, "BYSCORE")
+          transaction.zremrangebyscore(name, score, score)
+        }.first
+
+        if results.size == 1
+          yield results.first
+          @_size -= 1
+        else
+          # multiple jobs with the same score
+          # find the one with the right JID and push it
+          matched, nonmatched = results.partition { |message|
+            if message.index(jid)
+              msg = Sidekiq.load_json(message)
+              msg["jid"] == jid
+            else
+              false
+            end
+          }
+
+          msg = matched.first
+          if msg
+            yield msg
+            @_size -= 1
+          end
+
+          # push the rest back onto the sorted set
+          conn.multi do |transaction|
+            nonmatched.each do |message|
+              transaction.zadd(name, score.to_f.to_s, message)
+            end
+          end
+        end
+      end
+    end
+
     # :nodoc:
     # @api private
     def delete_by_value(name, value)
@@ -992,19 +1040,20 @@ module Sidekiq
         # you'll be happier this way
         conn.pipelined do |pipeline|
           procs.each do |key|
-            pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
+            pipeline.hmget(key, "info", "concurrency", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
 
-      result.each do |info, busy, beat, quiet, rss, rtt_us|
+      result.each do |info, concurrency, busy, beat, quiet, rss, rtt_us|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
         next if info.nil?
 
         hash = Sidekiq.load_json(info)
-        yield Process.new(hash.merge("busy" => busy.to_i,
+        yield Process.new(hash.merge("concurrency" => concurrency.to_i,
+          "busy" => busy.to_i,
           "beat" => beat.to_f,
           "quiet" => quiet,
           "rss" => rss.to_i,
@@ -1088,6 +1137,7 @@ module Sidekiq
     def identity
       self["identity"]
     end
+    alias_method :id, :identity
 
     # deprecated, use capsules below
     def queues
@@ -1161,6 +1211,10 @@ module Sidekiq
       self["quiet"] == "true"
     end
 
+    def leader?
+      Sidekiq.redis { |c| c.get("dear-leader") == identity }
+    end
+
     private
 
     def signal(sig)

data/lib/sidekiq/client.rb

@@ -132,12 +132,14 @@ module Sidekiq
     #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
     #
     def push_bulk(items)
-      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
       at = items.delete("at") || items.delete(:at)
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
+      # Use a smaller batch size by default for scheduled jobs since adding to sorted sets is more costly.
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || (at ? 100 : 1_000)
+
       jid = items.delete("jid")
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 

data/lib/sidekiq/component.rb

@@ -57,6 +57,9 @@ module Sidekiq
     end
 
     def tid
+      # We XOR with PID to ensure Thread IDs changes after fork.
+      # I'm unclear why we don't multiply the two values to better guarantee
+      # a unique value but it's been this way for quite a while now. #3685
       Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
     end
 

data/lib/sidekiq/launcher.rb

@@ -176,6 +176,7 @@ module Sidekiq
             transaction.sadd("processes", [key])
             transaction.exists(key)
             transaction.hset(key, "info", to_json,
+              "concurrency", @config.total_concurrency,
               "busy", curstate.size,
               "beat", Time.now.to_f,
               "rtt_us", rtt,
@@ -257,7 +258,6 @@ module Sidekiq
         "started_at" => Time.now.to_f,
         "pid" => ::Process.pid,
         "tag" => @config[:tag] || "",
-        "concurrency" => @config.total_concurrency,
         "capsules" => @config.capsules.each_with_object({}) { |(name, cap), memo|
           memo[name] = cap.to_h
         },

data/lib/sidekiq/manager.rb

@@ -69,7 +69,7 @@ module Sidekiq
     def processor_result(processor, reason = nil)
       @plock.synchronize do
         @workers.delete(processor)
-        unless @done
+        if !@done && @count > @workers.size
           p = Processor.new(@config, &method(:processor_result))
           @workers << p
           p.start

data/lib/sidekiq/paginator.rb

@@ -62,7 +62,12 @@ module Sidekiq
       pageidx = current_page - 1
       starting = pageidx * page_size
       items = items.to_a
-      [current_page, items.size, items[starting, page_size]]
+      total_size = items.size
+      if starting > total_size
+        starting = 0
+        current_page = 1
+      end
+      [current_page, total_size, items[starting, page_size]]
     end
   end
 end

data/lib/sidekiq/profiler.rb

@@ -21,7 +21,7 @@ module Sidekiq
       return yield unless job["profile"]
 
       token = job["profile"]
-      type = job["class"]
+      type = job["wrapped"] || job["class"]
       jid = job["jid"]
       started_at = Time.now