sidekiq 8.1.0 → 8.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63658318932ac6b7045211590b07f84ed9e1e6398f908ace790a0df20d5ac1f0
-  data.tar.gz: 0c3f5e69ada7529bdac7392acb1c56ef2a19b8ed13210be8c36aac19858c6df4
+  metadata.gz: 4fb090d79e2cf2b320fdd47ee55a574a68a46b77a23db27b849ddb6f54acb00a
+  data.tar.gz: 4863ef28ecdd8ad2cd868caf2f47cc9302b8eb88f4ff9da4f2b5641beca50d88
 SHA512:
-  metadata.gz: 9a5e95a2faccbbf42f8e651c4882c9495ad72b0500bb8fe77fa2a93fb4aa6f8cb96fc820621cd1910fb4b00accf35d75a99d548b3cc8ccdd41a18b5620a9ad73
-  data.tar.gz: 1bc66490ee07c548c71f680fb280e876efa7eeb8c6f234277df73211ce05c5ae29f5c9bc13064972da945568d1df6634252a6c22a7c5298bdd436b7266246c53
+  metadata.gz: 4ba71a2c43cec613337119cd29830339e213a8f876db4db84c62ac67890ee020ab841a4f7fe478f7f0fcd135571b885e54c9af12998ea6947ac51b543d52b42c
+  data.tar.gz: 9cabfc243aacec3c57c697b83374a5b934cc985c1e4ee86185a6faba7fe18f6540e3fa0b203c5dbfc0e22c03513f66e973be4152d8c8e12db29e97e21a94360f

data/Changes.md

@@ -2,6 +2,20 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.1
+----------
+
+- Add new `Sidekiq.testing!(mode)` API [#6931]
+Requiring code should not enable process-wide changes.
+```ruby
+# Old, implicit
+require "sidekiq/testing"
+# New, more explicit
+Sidekiq.testing!(:fake)
+```
+- Fix race condition with Stop button in UI [#6935]
+- Fix javascript error handler [#6893]
+
 8.1.0
 ----------
 

data/bin/tui

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/sidekiq/tui"
+
+Sidekiq::TUI.new.run

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -61,10 +61,13 @@ begin
 
         # @api private
         def enqueue(job)
-          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
+          # NB: Active Job only serializes keys it recognizes. We
+          # cannot set arbitrary key/values here.
+          wrapper = Sidekiq::ActiveJob::Wrapper.set(
             wrapped: job.class,
             queue: job.queue_name
-          ).perform_async(job.serialize)
+          )
+          job.provider_job_id = wrapper.perform_async(job.serialize)
         end
 
         # @api private

data/lib/generators/sidekiq/job_generator.rb

@@ -14,7 +14,10 @@ module Sidekiq
       end
 
       def create_job_file
-        template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
+        template(
+          "job.rb.erb",
+          File.join(jobs_directory, class_path, "#{file_name}_job.rb")
+        )
       end
 
       def create_test_file
@@ -31,7 +34,8 @@ module Sidekiq
 
       def create_job_spec
         template_file = File.join(
-          "spec/sidekiq",
+          "spec",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_spec.rb"
         )
@@ -40,7 +44,8 @@ module Sidekiq
 
       def create_job_test
         template_file = File.join(
-          "test/sidekiq",
+          "test",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_test.rb"
         )
@@ -54,6 +59,13 @@ module Sidekiq
       def test_framework
         ::Rails.application.config.generators.options[:rails][:test_framework]
       end
+
+      # Can be set in an initializer or in application configuration
+      # with Rails.application.config.generators.options[:sidekiq][:jobs_directory] = "app/jobs"
+      # to change the directory that the job files are generated in to.
+      def jobs_directory
+        ::Rails.application.config.generators.options[:sidekiq].fetch(:jobs_directory, "app/sidekiq")
+      end
     end
   end
 end

data/lib/sidekiq/api.rb

@@ -17,12 +17,37 @@ require "sidekiq/metrics/query"
 #
 
 module Sidekiq
+  module ApiUtils
+    # @api private
+    # Calculate the latency in seconds for a job based on its enqueued timestamp
+    # @param job [Hash] the job hash
+    # @return [Float] latency in seconds
+    def calculate_latency(job)
+      timestamp = job["enqueued_at"] || job["created_at"]
+      return 0.0 unless timestamp
+
+      if timestamp.is_a?(Float)
+        # old format
+        Time.now.to_f - timestamp
+      else
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+        (now - timestamp) / 1000.0
+      end
+    end
+  end
+
   # Retrieve runtime statistics from Redis regarding
   # this Sidekiq cluster.
   #
   #   stat = Sidekiq::Stats.new
   #   stat.processed
   class Stats
+    QueueSummary = Data.define(:name, :size, :latency, :paused) do
+      alias_method :paused?, :paused
+    end
+
+    include ApiUtils
+
     def initialize
       fetch_stats_fast!
     end
@@ -63,6 +88,7 @@ module Sidekiq
       stat :default_queue_latency
     end
 
+    # @return [Hash{String => Integer}] a hash of queue names to their lengths
     def queues
       Sidekiq.redis do |conn|
         queues = conn.sscan("queues").to_a
@@ -78,6 +104,41 @@ module Sidekiq
       end
     end
 
+    # More detailed information about each queue: name, size, latency, paused status
+    # @return [Array<QueueSummary>]
+    def queue_summaries
+      Sidekiq.redis do |conn|
+        queues = conn.sscan("queues").to_a
+        return [] if queues.empty?
+
+        results = conn.pipelined { |pipeline|
+          queues.each do |queue|
+            pipeline.llen("queue:#{queue}")
+            pipeline.lindex("queue:#{queue}", -1)
+            pipeline.sismember("paused", queue)
+          end
+        }
+
+        queue_summaries = []
+        queues.each_with_index do |name, idx|
+          size = results[idx * 3]
+          last_item = results[idx * 3 + 1]
+          paused = results[idx * 3 + 2] > 0
+
+          latency = if last_item
+            job = Sidekiq.load_json(last_item)
+            calculate_latency(job)
+          else
+            0.0
+          end
+
+          queue_summaries << QueueSummary.new(name:, size:, latency:, paused:)
+        end
+
+        queue_summaries.sort_by { |qd| -qd.size }
+      end
+    end
+
     # O(1) redis calls
     # @api private
     def fetch_stats_fast!
@@ -100,19 +161,7 @@ module Sidekiq
           {}
         end
 
-        enqueued_at = job["enqueued_at"]
-        if enqueued_at
-          if enqueued_at.is_a?(Float)
-            # old format
-            now = Time.now.to_f
-            now - enqueued_at
-          else
-            now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-            (now - enqueued_at) / 1000.0
-          end
-        else
-          0.0
-        end
+        calculate_latency(job)
       else
         0.0
       end
@@ -235,6 +284,7 @@ module Sidekiq
   #   end
   class Queue
     include Enumerable
+    include ApiUtils
 
     ##
     # Fetch all known queues within Redis.
@@ -245,6 +295,7 @@ module Sidekiq
     end
 
     attr_reader :name
+    alias_method :id, :name
 
     # @param name [String] the name of the queue
     def initialize(name = "default")
@@ -277,19 +328,7 @@ module Sidekiq
       return 0.0 unless entry
 
       job = Sidekiq.load_json(entry)
-      enqueued_at = job["enqueued_at"]
-      if enqueued_at
-        if enqueued_at.is_a?(Float)
-          # old format
-          now = Time.now.to_f
-          now - enqueued_at
-        else
-          now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-          (now - enqueued_at) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(job)
     end
 
     def each
@@ -352,6 +391,8 @@ module Sidekiq
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
   class JobRecord
+    include ApiUtils
+
     # the parsed Hash of job data
     # @!attribute [r] Item
     attr_reader :item
@@ -478,17 +519,7 @@ module Sidekiq
     end
 
     def latency
-      timestamp = @item["enqueued_at"] || @item["created_at"]
-      if timestamp
-        if timestamp.is_a?(Float)
-          # old format
-          Time.now.to_f - timestamp
-        else
-          (::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) - timestamp) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(@item)
     end
 
     # Remove this job from the queue
@@ -553,17 +584,24 @@ module Sidekiq
   # could be the scheduled time for it to run (e.g. scheduled set),
   # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
-    attr_reader :score
     attr_reader :parent
 
     # :nodoc:
     # @api private
     def initialize(parent, score, item)
       super(item)
-      @score = Float(score)
+      @score = score
       @parent = parent
     end
 
+    def score
+      Float(@score)
+    end
+
+    def id
+      "#{@score}|#{item["jid"]}"
+    end
+
     # The timestamp associated with this entry
     def at
       Time.at(score).utc
@@ -574,7 +612,7 @@ module Sidekiq
       if @value
         @parent.delete_by_value(@parent.name, @value)
       else
-        @parent.delete_by_jid(score, jid)
+        @parent.delete_by_jid(@score, jid)
       end
     end
 
@@ -583,7 +621,7 @@ module Sidekiq
     # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
-        conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
+        conn.zincrby(@parent.name, at.to_f - score, Sidekiq.dump_json(@item))
       end
     end
 
@@ -1088,6 +1126,7 @@ module Sidekiq
     def identity
       self["identity"]
     end
+    alias_method :id, :identity
 
     # deprecated, use capsules below
     def queues
@@ -1161,6 +1200,10 @@ module Sidekiq
       self["quiet"] == "true"
     end
 
+    def leader?
+      Sidekiq.redis { |c| c.get("dear-leader") == identity }
+    end
+
     private
 
     def signal(sig)

data/lib/sidekiq/test_api.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq"
+
+module Sidekiq
+  class Testing
+    class TestModeAlreadySetError < RuntimeError; end
+    class << self
+      attr_accessor :__global_test_mode
+
+      # Calling without a block sets the global test mode, affecting
+      # all threads. Calling with a block only affects the current Thread.
+      def __set_test_mode(mode)
+        if block_given?
+          # Reentrant testing modes will lead to a rat's nest of code which is
+          # hard to reason about. You can set the testing mode once globally and
+          # you can override that global setting once per-thread.
+          raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
+
+          self.__local_test_mode = mode
+          begin
+            yield
+          ensure
+            self.__local_test_mode = nil
+          end
+        else
+          self.__global_test_mode = mode
+        end
+      end
+
+      def __test_mode
+        __local_test_mode || __global_test_mode
+      end
+
+      def __local_test_mode
+        Thread.current[:__sidekiq_test_mode]
+      end
+
+      def __local_test_mode=(value)
+        Thread.current[:__sidekiq_test_mode] = value
+      end
+
+      def disable!(&block)
+        __set_test_mode(:disable, &block)
+      end
+
+      def fake!(&block)
+        __set_test_mode(:fake, &block)
+      end
+
+      def inline!(&block)
+        __set_test_mode(:inline, &block)
+      end
+
+      def enabled?
+        __test_mode != :disable
+      end
+
+      def disabled?
+        __test_mode == :disable
+      end
+
+      def fake?
+        __test_mode == :fake
+      end
+
+      def inline?
+        __test_mode == :inline
+      end
+
+      def server_middleware
+        @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
+        yield @server_chain if block_given?
+        @server_chain
+      end
+    end
+  end
+
+  class EmptyQueueError < RuntimeError; end
+
+  module TestingClient
+    private def atomic_push(conn, payloads)
+      if Sidekiq::Testing.fake?
+        payloads.each do |job|
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job["enqueued_at"] = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) unless job["at"]
+          Queues.push(job["queue"], job["class"], job)
+        end
+        true
+      elsif Sidekiq::Testing.inline?
+        payloads.each do |job|
+          klass = Object.const_get(job["class"])
+          job["id"] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
+        end
+        true
+      else
+        super
+      end
+    end
+  end
+
+  Sidekiq::Client.prepend TestingClient
+
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # There are 2 data structures involved in tandem. This is due to the
+    # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
+    # to the array. Because the array was derived from a filter of the total
+    # jobs enqueued, it appeared as though the array didn't change.
+    #
+    # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
+    # on the queue, and another with keys of the job type, so the array for
+    # HardJob.jobs is a straight reference to a real array.
+    #
+    # Queue-based hash:
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::HardJob",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Job-based hash:
+    #
+    # {
+    #   "TestTesting::HardJob"=>[
+    #     {
+    #       "class"=>"TestTesting::HardJob",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardJob.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs_by_queue[queue]
+      end
+
+      def push(queue, klass, job)
+        jobs_by_queue[queue] << job
+        jobs_by_class[klass] << job
+      end
+
+      def jobs_by_queue
+        @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def jobs_by_class
+        @jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+      alias_method :jobs_by_worker, :jobs_by_class
+
+      def delete_for(jid, queue, klass)
+        jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
+        jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
+      end
+
+      def clear_for(queue, klass)
+        jobs_by_queue[queue.to_s].clear
+        jobs_by_class[klass].clear
+      end
+
+      def clear_all
+        jobs_by_queue.clear
+        jobs_by_class.clear
+      end
+    end
+  end
+
+  module Job
+    ##
+    # The Sidekiq testing infrastructure overrides perform_async
+    # so that it does not actually touch the network.  Instead it
+    # stores the asynchronous jobs in a per-class array so that
+    # their presence/absence can be asserted by your tests.
+    #
+    # This is similar to ActionMailer's :test delivery_method and its
+    # ActionMailer::Base.deliveries array.
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, HardJob.jobs.size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, HardJob.jobs.size
+    #   assert_equal :something, HardJob.jobs[0]['args'][0]
+    #
+    # You can also clear and drain all job types:
+    #
+    #   Sidekiq::Job.clear_all # or .drain_all
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Job.clear_all
+    #     end
+    #   end
+    #
+    # or for acceptance testing, i.e. with cucumber:
+    #
+    #   AfterStep do
+    #     Sidekiq::Job.drain_all
+    #   end
+    #
+    #   When I sign up as "foo@example.com"
+    #   Then I should receive a welcome email to "foo@example.com"
+    #
+    module ClassMethods
+      # Queue for this worker
+      def queue
+        get_sidekiq_options["queue"]
+      end
+
+      # Jobs queued for this worker
+      def jobs
+        Queues.jobs_by_class[to_s]
+      end
+
+      # Clear all jobs for this worker
+      def clear
+        Queues.clear_for(queue, to_s)
+      end
+
+      # Drain and run all jobs for this worker
+      def drain
+        while jobs.any?
+          next_job = jobs.first
+          Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
+          process_job(next_job)
+        end
+      end
+
+      # Pop out a single job and perform it
+      def perform_one
+        raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
+        next_job = jobs.first
+        Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
+        process_job(next_job)
+      end
+
+      def process_job(job)
+        instance = new
+        instance.jid = job["jid"]
+        instance.bid = job["bid"] if instance.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(instance, job, job["queue"]) do
+          execute_job(instance, job["args"])
+        end
+      end
+
+      def execute_job(worker, args)
+        worker.perform(*args)
+      end
+    end
+
+    class << self
+      def jobs # :nodoc:
+        Queues.jobs_by_queue.values.flatten
+      end
+
+      # Clear all queued jobs
+      def clear_all
+        Queues.clear_all
+      end
+
+      # Drain (execute) all queued jobs
+      def drain_all
+        while jobs.any?
+          job_classes = jobs.map { |job| job["class"] }.uniq
+
+          job_classes.each do |job_class|
+            Object.const_get(job_class).drain
+          end
+        end
+      end
+    end
+  end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
+      end
+    end
+  end
+end
+
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING # rubocop:disable Style/GlobalVars
+  warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
+end

data/lib/sidekiq/testing/inline.rb

@@ -1,30 +1,2 @@
-# frozen_string_literal: true
-
-require "sidekiq/testing"
-
-##
-# The Sidekiq inline infrastructure overrides perform_async so that it
-# actually calls perform instead. This allows jobs to be run inline in a
-# testing environment.
-#
-# This is similar to `Resque.inline = true` functionality.
-#
-# Example:
-#
-#   require 'sidekiq/testing/inline'
-#
-#   $external_variable = 0
-#
-#   class ExternalJob
-#     include Sidekiq::Job
-#
-#     def perform
-#       $external_variable = 1
-#     end
-#   end
-#
-#   assert_equal 0, $external_variable
-#   ExternalJob.perform_async
-#   assert_equal 1, $external_variable
-#
-Sidekiq::Testing.inline!
+Sidekiq.testing!(:inline)
+warn('⛔️ `require "sidekiq/testing/inline"` is deprecated and will be removed in Sidekiq 9.0. See https://sidekiq.org/wiki/Testing#new-api')