sidekiq 7.3.0 → 7.3.9

data/lib/sidekiq/job/iterable.rb

@@ -30,6 +30,40 @@ module Sidekiq
         @_cursor = nil
         @_start_time = nil
         @_runtime = 0
+        @_args = nil
+        @_cancelled = nil
+      end
+
+      def arguments
+        @_args
+      end
+
+      # Three days is the longest period you generally need to wait for a retry to
+      # execute when using the default retry scheme. We don't want to "forget" the job
+      # is cancelled before it has a chance to execute and cancel itself.
+      CANCELLATION_PERIOD = (3 * 86_400).to_s
+
+      # Set a flag in Redis to mark this job as cancelled.
+      # Cancellation is asynchronous and is checked at the start of iteration
+      # and every 5 seconds thereafter as part of the recurring state flush.
+      def cancel!
+        return @_cancelled if cancelled?
+
+        key = "it-#{jid}"
+        _, result, _ = Sidekiq.redis do |c|
+          c.pipelined do |p|
+            p.hsetnx(key, "cancelled", Time.now.to_i)
+            p.hget(key, "cancelled")
+            # TODO When Redis 7.2 is required
+            # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+            p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
+          end
+        end
+        @_cancelled = result.to_i
+      end
+
+      def cancelled?
+        @_cancelled
       end
 
       # A hook to override that will be called when the job starts iterating.
@@ -91,13 +125,14 @@ module Sidekiq
       end
 
       # @api private
-      def perform(*arguments)
+      def perform(*args)
+        @_args = args.dup.freeze
         fetch_previous_iteration_state
 
         @_executions += 1
         @_start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
 
-        enumerator = build_enumerator(*arguments, cursor: @_cursor)
+        enumerator = build_enumerator(*args, cursor: @_cursor)
         unless enumerator
           logger.info("'#build_enumerator' returned nil, skipping the job.")
           return
@@ -112,7 +147,7 @@ module Sidekiq
         end
 
         completed = catch(:abort) do
-          iterate_with_enumerator(enumerator, arguments)
+          iterate_with_enumerator(enumerator, args)
         end
 
         on_stop
@@ -128,6 +163,10 @@ module Sidekiq
 
       private
 
+      def is_cancelled?
+        @_cancelled = Sidekiq.redis { |c| c.hget("it-#{jid}", "cancelled") }
+      end
+
       def fetch_previous_iteration_state
         state = Sidekiq.redis { |conn| conn.hgetall(iteration_key) }
 
@@ -144,6 +183,12 @@ module Sidekiq
       STATE_TTL = 30 * 24 * 60 * 60 # one month
 
       def iterate_with_enumerator(enumerator, arguments)
+        if is_cancelled?
+          logger.info { "Job cancelled" }
+          return true
+        end
+
+        time_limit = Sidekiq.default_configuration[:timeout]
         found_record = false
         state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
 
@@ -153,14 +198,21 @@ module Sidekiq
 
           is_interrupted = interrupted?
           if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - state_flushed_at >= STATE_FLUSH_INTERVAL || is_interrupted
-            flush_state
+            _, _, cancelled = flush_state
             state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+            if cancelled
+              @_cancelled = true
+              logger.info { "Job cancelled" }
+              return true
+            end
           end
 
           return false if is_interrupted
 
-          around_iteration do
-            each_iteration(object, *arguments)
+          verify_iteration_time(time_limit, object) do
+            around_iteration do
+              each_iteration(object, *arguments)
+            end
           end
         end
 
@@ -170,6 +222,16 @@ module Sidekiq
         @_runtime += (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @_start_time)
       end
 
+      def verify_iteration_time(time_limit, object)
+        start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        yield
+        finish = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        total = finish - start
+        if total > time_limit
+          logger.warn { "Iteration took longer (%.2f) than Sidekiq's shutdown timeout (%d) when processing `%s`. This can lead to job processing problems during deploys" % [total, time_limit, object] }
+        end
+      end
+
       def reenqueue_iteration_job
         flush_state
         logger.debug { "Interrupting job (cursor=#{@_cursor.inspect})" }
@@ -204,6 +266,7 @@ module Sidekiq
           conn.multi do |pipe|
             pipe.hset(key, state)
             pipe.expire(key, STATE_TTL)
+            pipe.hget(key, "cancelled")
           end
         end
       end

data/lib/sidekiq/job_logger.rb

@@ -2,36 +2,24 @@
 
 module Sidekiq
   class JobLogger
-    include Sidekiq::Component
-
     def initialize(config)
       @config = config
-      @logger = logger
-    end
-
-    # If true we won't do any job logging out of the box.
-    # The user is responsible for any logging.
-    def skip_default_logging?
-      config[:skip_default_job_logging]
+      @logger = @config.logger
+      @skip = !!@config[:skip_default_job_logging]
     end
 
     def call(item, queue)
-      return yield if skip_default_logging?
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      @logger.info { "start" } unless @skip
 
-      begin
-        start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-        @logger.info("start")
+      yield
 
-        yield
-
-        Sidekiq::Context.add(:elapsed, elapsed(start))
-        @logger.info("done")
-      rescue Exception
-        Sidekiq::Context.add(:elapsed, elapsed(start))
-        @logger.info("fail")
-
-        raise
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info { "done" } unless @skip
+    rescue Exception
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info { "fail" } unless @skip
+      raise
     end
 
     def prepare(job_hash, &block)

data/lib/sidekiq/job_util.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "securerandom"
 require "time"
 

data/lib/sidekiq/launcher.rb

@@ -36,8 +36,8 @@ module Sidekiq
     # has a heartbeat thread, caller can use `async_beat: false`
     # and instead have thread call Launcher#heartbeat every N seconds.
     def run(async_beat: true)
-      Sidekiq.freeze!
       logger.debug { @config.merge!({}) }
+      Sidekiq.freeze!
       @thread = safe_thread("heartbeat", &method(:start_heartbeat)) if async_beat
       @poller.start
       @managers.each(&:start)

data/lib/sidekiq/metrics/query.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq"
 require "date"
 require "set"

data/lib/sidekiq/metrics/shared.rb

@@ -1,10 +1,21 @@
-require "concurrent"
+# frozen_string_literal: true
 
 module Sidekiq
   module Metrics
-    # This is the only dependency on concurrent-ruby in Sidekiq but it's
-    # mandatory for thread-safety until MRI supports atomic operations on values.
-    Counter = ::Concurrent::AtomicFixnum
+    class Counter
+      def initialize
+        @value = 0
+        @lock = Mutex.new
+      end
+
+      def increment
+        @lock.synchronize { @value += 1 }
+      end
+
+      def value
+        @lock.synchronize { @value }
+      end
+    end
 
     # Implements space-efficient but statistically useful histogram storage.
     # A precise time histogram stores every time. Instead we break times into a set of

data/lib/sidekiq/metrics/tracking.rb

@@ -31,11 +31,11 @@ module Sidekiq
           # We don't track time for failed jobs as they can have very unpredictable
           # execution times. more important to know average time for successful jobs so we
           # can better recognize when a perf regression is introduced.
-          @lock.synchronize {
-            @grams[klass].record_time(time_ms)
-            @jobs["#{klass}|ms"] += time_ms
-            @totals["ms"] += time_ms
-          }
+          track_time(klass, time_ms)
+        rescue JobRetry::Skip
+          # This is raised when iterable job is interrupted.
+          track_time(klass, time_ms)
+          raise
         rescue Exception
           @lock.synchronize {
             @jobs["#{klass}|f"] += 1
@@ -100,6 +100,14 @@ module Sidekiq
 
       private
 
+      def track_time(klass, time_ms)
+        @lock.synchronize {
+          @grams[klass].record_time(time_ms)
+          @jobs["#{klass}|ms"] += time_ms
+          @totals["ms"] += time_ms
+        }
+      end
+
       def reset
         @lock.synchronize {
           array = [@totals, @jobs, @grams]

data/lib/sidekiq/middleware/current_attributes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "active_support/current_attributes"
 
 module Sidekiq
@@ -31,11 +33,26 @@ module Sidekiq
             attrs = strklass.constantize.attributes
             # Retries can push the job N times, we don't
             # want retries to reset cattr. #5692, #5090
-            job[key] = attrs if attrs.any?
+            if attrs.any?
+              # Older rails has a bug that `CurrentAttributes#attributes` always returns
+              # the same hash instance. We need to dup it to avoid being accidentally mutated.
+              job[key] = if returns_same_object?
+                attrs.dup
+              else
+                attrs
+              end
+            end
           end
         end
         yield
       end
+
+      private
+
+      def returns_same_object?
+        ActiveSupport::VERSION::MAJOR < 8 ||
+          (ActiveSupport::VERSION::MAJOR == 8 && ActiveSupport::VERSION::MINOR == 0)
+      end
     end
 
     class Load
@@ -86,7 +103,7 @@ module Sidekiq
         cattrs = build_cattrs_hash(klass_or_array)
 
         config.client_middleware.add Save, cattrs
-        config.server_middleware.add Load, cattrs
+        config.server_middleware.prepend Load, cattrs
       end
 
       private

data/lib/sidekiq/middleware/modules.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sidekiq
   # Server-side middleware must import this Module in order
   # to get access to server resources during `call`.

data/lib/sidekiq/monitor.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "fileutils"
 require "sidekiq/api"
@@ -98,7 +99,7 @@ class Sidekiq::Monitor
       pad = opts[:pad] || 0
       max_length = opts[:max_length] || (80 - pad)
       out = []
-      line = ""
+      line = +""
       values.each do |value|
         if (line.length + value.length) > max_length
           out << line

data/lib/sidekiq/paginator.rb

@@ -2,6 +2,12 @@
 
 module Sidekiq
   module Paginator
+    TYPE_CACHE = {
+      "dead" => "zset",
+      "retry" => "zset",
+      "schedule" => "zset"
+    }
+
     def page(key, pageidx = 1, page_size = 25, opts = nil)
       current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
       pageidx = current_page - 1

data/lib/sidekiq/processor.rb

@@ -138,11 +138,11 @@ module Sidekiq
               # Effectively this block denotes a "unit of work" to Rails.
               @reloader.call do
                 klass = Object.const_get(job_hash["class"])
-                inst = klass.new
-                inst.jid = job_hash["jid"]
-                inst._context = self
-                @retrier.local(inst, jobstr, queue) do
-                  yield inst
+                instance = klass.new
+                instance.jid = job_hash["jid"]
+                instance._context = self
+                @retrier.local(instance, jobstr, queue) do
+                  yield instance
                 end
               end
             end
@@ -180,9 +180,9 @@ module Sidekiq
       ack = false
       Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
         Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
-          dispatch(job_hash, queue, jobstr) do |inst|
-            config.server_middleware.invoke(inst, job_hash, queue) do
-              execute_job(inst, job_hash["args"])
+          dispatch(job_hash, queue, jobstr) do |instance|
+            config.server_middleware.invoke(instance, job_hash, queue) do
+              execute_job(instance, job_hash["args"])
             end
           end
           ack = true
@@ -216,8 +216,8 @@ module Sidekiq
       end
     end
 
-    def execute_job(inst, cloned_args)
-      inst.perform(*cloned_args)
+    def execute_job(instance, cloned_args)
+      instance.perform(*cloned_args)
     end
 
     # Ruby doesn't provide atomic counters out of the box so we'll

data/lib/sidekiq/rails.rb

@@ -4,6 +4,17 @@ require "sidekiq/job"
 require "rails"
 
 module Sidekiq
+  module ActiveJob
+    # @api private
+    class Wrapper
+      include Sidekiq::Job
+
+      def perform(job_data)
+        ::ActiveJob::Base.execute(job_data.merge("provider_job_id" => jid))
+      end
+    end
+  end
+
   class Rails < ::Rails::Engine
     class Reloader
       def initialize(app = ::Rails.application)
@@ -39,6 +50,7 @@ module Sidekiq
     #   end
     initializer "sidekiq.active_job_integration" do
       ActiveSupport.on_load(:active_job) do
+        require_relative "../active_job/queue_adapters/sidekiq_adapter"
         include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
       end
     end

data/lib/sidekiq/redis_connection.rb

@@ -66,7 +66,14 @@ module Sidekiq
         scrubbed_options[:password] = redacted if scrubbed_options[:password]
         scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
         scrubbed_options[:sentinels]&.each do |sentinel|
-          sentinel[:password] = redacted if sentinel[:password]
+          if sentinel.is_a?(String)
+            if (uri = URI(sentinel)) && uri.password
+              uri.password = redacted
+              sentinel.replace(uri.to_s)
+            end
+          elsif sentinel[:password]
+            sentinel[:password] = redacted
+          end
         end
         scrubbed_options
       end

data/lib/sidekiq/ring_buffer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module Sidekiq

data/lib/sidekiq/systemd.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Sidekiq's systemd integration allows Sidekiq to inform systemd:
 #  1. when it has successfully started

data/lib/sidekiq/testing.rb

@@ -283,11 +283,11 @@ module Sidekiq
       end
 
       def process_job(job)
-        inst = new
-        inst.jid = job["jid"]
-        inst.bid = job["bid"] if inst.respond_to?(:bid=)
-        Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
-          execute_job(inst, job["args"])
+        instance = new
+        instance.jid = job["jid"]
+        instance.bid = job["bid"] if instance.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(instance, job, job["queue"]) do
+          execute_job(instance, job["args"])
         end
       end
 

data/lib/sidekiq/version.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.3.0"
+  VERSION = "7.3.9"
   MAJOR = 7
+
+  def self.gem_version
+    Gem::Version.new(VERSION)
+  end
 end

data/lib/sidekiq/web/action.rb

@@ -27,6 +27,7 @@ module Sidekiq
       redirect current_location
     end
 
+    # deprecated, will warn in 8.0
     def params
       indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
 
@@ -36,8 +37,19 @@ module Sidekiq
       indifferent_hash
     end
 
-    def route_params
-      env[WebRouter::ROUTE_PARAMS]
+    # Use like `url_params("page")` within your action blocks
+    def url_params(key)
+      request.params[key]
+    end
+
+    # Use like `route_params(:name)` within your action blocks
+    # key is required in 8.0, nil is only used for backwards compatibility
+    def route_params(key = nil)
+      if key
+        env[WebRouter::ROUTE_PARAMS][key]
+      else
+        env[WebRouter::ROUTE_PARAMS]
+      end
     end
 
     def session
@@ -48,8 +60,12 @@ module Sidekiq
       if content.is_a? Symbol
         unless respond_to?(:"_erb_#{content}")
           views = options[:views] || Web.settings.views
-          src = ERB.new(File.read("#{views}/#{content}.erb")).src
-          WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          filename = "#{views}/#{content}.erb"
+          src = ERB.new(File.read(filename)).src
+
+          # Need to use lineno less by 1 because erb generates a
+          # comment before the source code.
+          WebAction.class_eval <<-RUBY, filename, -1 # standard:disable Style/EvalWithLocation
             def _erb_#{content}
               #{src}
             end