sidekiq-unique-jobs 7.0.2 → 7.1.12

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -10,25 +10,36 @@ module SidekiqUniqueJobs
     module Manager
       module_function
 
+      #
+      # @return [Float] the amount to add to the reaper interval
       DRIFT_FACTOR = 0.02
+      #
+      # @return [Symbol] allowed reapers (:ruby or :lua)
       REAPERS      = [:ruby, :lua].freeze
 
+      # includes "SidekiqUniqueJobs::Connection"
+      # @!parse include SidekiqUniqueJobs::Connection
       include SidekiqUniqueJobs::Connection
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::Logging
 
       #
       # Starts a separate thread that periodically reaps orphans
       #
       #
-      # @return [Concurrent::TimerTask] the task that was started
+      # @return [SidekiqUniqueJobs::TimerTask] the task that was started
       #
-      def start # rubocop:disable
+      def start(test_task = nil) # rubocop:disable
         return if disabled?
         return if registered?
 
+        self.task = test_task || default_task
+
         with_logging_context do
           register_reaper_process
           log_info("Starting Reaper")
+
           task.add_observer(Observer.new)
           task.execute
           task
@@ -59,7 +70,17 @@ module SidekiqUniqueJobs
       # @return [<type>] <description>
       #
       def task
-        @task ||= Concurrent::TimerTask.new(timer_task_options) do
+        @task ||= default_task
+      end
+
+      #
+      # A properly configured timer task
+      #
+      #
+      # @return [SidekiqUniqueJobs::TimerTask]
+      #
+      def default_task
+        SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
           with_logging_context do
             redis do |conn|
               refresh_reaper_mutex
@@ -69,6 +90,17 @@ module SidekiqUniqueJobs
         end
       end
 
+      #
+      # Store a task to use for scheduled execution
+      #
+      # @param [SidekiqUniqueJobs::TimerTask] task the task to use
+      #
+      # @return [void]
+      #
+      def task=(task)
+        @task = task
+      end
+
       #
       # Arguments passed on to the timer task
       #
@@ -190,10 +222,24 @@ module SidekiqUniqueJobs
         redis { |conn| conn.del(UNIQUE_REAPER) }
       end
 
+      #
+      # Reaper interval with a little drift
+      #   Redis isn't exact enough so to give a little bufffer,
+      #   we add a tiny value to the reaper interval.
+      #
+      #
+      # @return [Integer] <description>
+      #
       def drift_reaper_interval
         reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
       end
 
+      #
+      # Current time (as integer value)
+      #
+      #
+      # @return [Integer]
+      #
       def current_timestamp
         Time.now.to_i
       end

data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    # Restarts orphan manager if it is considered dead
+    module ReaperResurrector
+      module_function
+
+      include SidekiqUniqueJobs::Connection
+      include SidekiqUniqueJobs::Logging
+
+      DRIFT_FACTOR = 0.1
+      REAPERS = [:ruby, :lua].freeze
+
+      #
+      # Starts reaper resurrector that watches orphans reaper
+      #
+      # @return [SidekiqUniqueJobs::TimerTask] the task that was started
+      #
+      def start
+        return if resurrector_disabled?
+        return if reaper_disabled?
+
+        with_logging_context do
+          run_task
+        end
+      end
+
+      #
+      # Runs reaper resurrector task
+      #
+      # @return [SidekiqUniqueJobs::TimerTask]
+      def run_task
+        log_info("Starting Reaper Resurrector")
+        task.execute
+        task
+      end
+
+      #
+      # The task that runs the resurrector
+      #
+      # @return [SidekiqUniqueJobs::TimerTask]
+      def task
+        SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
+          with_logging_context do
+            restart_if_dead
+          end
+        end
+      end
+
+      #
+      # Starts new instance of orphan reaper if reaper is considered dead (reaper mutex has not been refreshed lately)
+      #
+      def restart_if_dead
+        return if reaper_registered?
+
+        log_info("Reaper is considered dead. Starting new reaper instance")
+        orphans_manager.start
+      end
+
+      #
+      # Returns orphan manager
+      #
+      # @return [SidekiqUniqueJobs::Orphans::Manager]
+      def orphans_manager
+        SidekiqUniqueJobs::Orphans::Manager
+      end
+
+      #
+      # Checks if resurrector is disabled
+      #
+      # @see resurrector_enabled?
+      #
+      # @return [true, false]
+      def resurrector_disabled?
+        !resurrector_enabled?
+      end
+
+      #
+      # Checks if resurrector is enabled
+      #
+      # @return [true, false]
+      def resurrector_enabled?
+        SidekiqUniqueJobs.config.reaper_resurrector_enabled
+      end
+
+      #
+      # Checks if reaping is disabled
+      #
+      # @see reaper_enabled?
+      #
+      # @return [true, false]
+      #
+      def reaper_disabled?
+        !reaper_enabled?
+      end
+
+      #
+      # Checks if reaping is enabled
+      #
+      # @return [true, false]
+      #
+      def reaper_enabled?
+        REAPERS.include?(reaper)
+      end
+
+      #
+      # Checks if reaper is registered
+      #
+      # @return [true, false]
+      def reaper_registered?
+        redis do |conn|
+          conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
+        end
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper
+      #
+      def reaper
+        SidekiqUniqueJobs.config.reaper
+      end
+
+      #
+      # Arguments passed on to the timer task
+      #
+      #
+      # @return [Hash]
+      #
+      def timer_task_options
+        { run_now: false,
+          execution_interval: reaper_resurrector_interval }
+      end
+
+      #
+      # A context to use for all log entries
+      #
+      #
+      # @return [Hash] when logger responds to `:with_context`
+      # @return [String] when logger does not responds to `:with_context`
+      #
+      def logging_context
+        if logger_context_hash?
+          { "uniquejobs" => "reaper-resurrector" }
+        else
+          "uniquejobs=reaper-resurrector"
+        end
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper_resurrector_interval
+      #
+      def reaper_resurrector_interval
+        SidekiqUniqueJobs.config.reaper_resurrector_interval
+      end
+
+      def reaper_interval
+        SidekiqUniqueJobs.config.reaper_interval
+      end
+
+      def drift_reaper_interval
+        reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
+      end
+
+      def current_timestamp
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -9,7 +9,11 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
+    # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      #
+      # @return [String] the suffix for :RUN locks
+      RUN_SUFFIX = ":RUN"
       #
       # @!attribute [r] digests
       #   @return [SidekiqUniqueJobs::Digests] digest collection
@@ -51,13 +55,27 @@ module SidekiqUniqueJobs
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
-          next if belongs_to_job?(digest)
+      def orphans # rubocop:disable Metrics/MethodLength
+        page = 0
+        per = reaper_count * 2
+        orphans = []
+        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+
+        while results.size.positive?
+          results.each do |digest|
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
+
+          break if orphans.size >= reaper_count
 
-          memo << digest
-          break if memo.size >= reaper_count
+          page += 1
+          results = conn.zrange(digests.key, page * per, (page + 1) * per)
         end
+
+        orphans
       end
 
       #
@@ -124,7 +142,12 @@ module SidekiqUniqueJobs
 
           procs.sort.each do |key|
             valid, workers = conn.pipelined do
-              conn.exists(key)
+              # TODO: Remove the if statement in the future
+              if conn.respond_to?(:exists?)
+                conn.exists?(key)
+              else
+                conn.exists(key)
+              end
               conn.hgetall("#{key}:workers")
             end
 
@@ -136,7 +159,7 @@ module SidekiqUniqueJobs
 
               payload = safe_load_json(item[PAYLOAD])
 
-              return true if payload[LOCK_DIGEST] == digest
+              return true if match?(digest, payload[LOCK_DIGEST])
               return true if considered_active?(payload[CREATED_AT])
             end
           end
@@ -145,6 +168,12 @@ module SidekiqUniqueJobs
         end
       end
 
+      def match?(key_one, key_two)
+        return false if key_one.nil? || key_two.nil?
+
+        key_one.delete_suffix(RUN_SUFFIX) == key_two.delete_suffix(RUN_SUFFIX)
+      end
+
       def considered_active?(time_f)
         (Time.now - reaper_timeout).to_f < time_f
       end
@@ -170,7 +199,7 @@ module SidekiqUniqueJobs
         page_size    = 50
 
         loop do
-          range_start = page * page_size - deleted_size
+          range_start = (page * page_size) - deleted_size
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -197,5 +226,6 @@ module SidekiqUniqueJobs
         conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/sidekiq_unique_jobs/redis/entity.rb

@@ -47,7 +47,13 @@ module SidekiqUniqueJobs
       #
       def exist?
         redis do |conn|
-          value = conn.exists(key)
+          # TODO: Remove the if statement in the future
+          value =
+            if conn.respond_to?(:exists?)
+              conn.exists?(key)
+            else
+              conn.exists(key)
+            end
 
           return value if boolean?(value)
 

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -26,7 +26,7 @@ module SidekiqUniqueJobs
       #
       # Adds a value to the sorted set
       #
-      # @param [Array<Float, String>, String] the values to add
+      # @param [Array<Float, String>, String] values the values to add
       #
       # @return [Boolean, Integer] <description>
       #

data/lib/sidekiq_unique_jobs/reflectable.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Module Reflectable provides a method to notify subscribers
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module Reflectable
+    #
+    # Reflects on specific event
+    #
+    # @param [Symbol] reflection the reflected event
+    # @param [Array] args arguments to provide to reflector
+    #
+    # @return [void]
+    #
+    def reflect(reflection, *args)
+      SidekiqUniqueJobs.reflections.dispatch(reflection, *args)
+      nil
+    rescue UniqueJobsError => ex
+      SidekiqUniqueJobs.logger.error(ex)
+      nil
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/reflections.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class NotificationCollection provides a collection with known notifications
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Reflections
+    #
+    # @return [Array<Symbol>] list of notifications
+    REFLECTIONS = [
+      :after_unlock_callback_failed,
+      :debug,
+      :duplicate,
+      :error,
+      :execution_failed,
+      :lock_failed,
+      :locked,
+      :reschedule_failed,
+      :rescheduled,
+      :timeout,
+      :unknown_sidekiq_worker,
+      :unlock_failed,
+      :unlocked,
+    ].freeze
+
+    #
+    # @return [Hash<Symbol, Array<Symbol, String>>] a hash with deprecated notifications
+    DEPRECATIONS = {}.freeze
+
+    REFLECTIONS.each do |reflection|
+      class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
+        def #{reflection}(*args, &block)                            # def unlock_failed(*args, &block)
+          raise NoBlockGiven, "block required" unless block_given?  #   raise NoBlockGiven, "block required" unless block_given?
+          @reflections[:#{reflection}] = block                      #   @notifications[:unlock_failed] = block
+        end                                                         # end
+      RUBY
+    end
+
+    def initialize
+      @reflections = {}
+    end
+
+    #
+    # Dispatch a reflected event
+    #
+    # @param [reflection] reflection the reflected event
+    # @param [Array] args the arguments to provide to the block
+    #
+    # @return [void] <description>
+    #
+    def dispatch(reflection, *args)
+      if (block = @reflections[reflection])
+        block.call(*args)
+
+        if DEPRECATIONS.key?(reflection)
+          replacement, removal_version = DEPRECATIONS[reflection]
+          SidekiqUniqueJobs::Deprecation.warn(
+            "#{reflection} is deprecated and will be removed in version #{removal_version}." \
+            " Use #{replacement} instead.",
+          )
+        end
+      elsif misconfigured?(reflection)
+        raise NoSuchNotificationError, reflection
+      end
+
+      nil
+    end
+
+    def configured?(reflection)
+      REFLECTIONS.include?(reflection)
+    end
+
+    def misconfigured?(reflection)
+      !configured?(reflection)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -45,7 +45,9 @@ module SidekiqUniqueJobs
         pool = defined?(redis_pool) ? redis_pool : nil
 
         redis(pool) do |new_conn|
-          do_call(file_name, new_conn, keys, argv)
+          result = do_call(file_name, new_conn, keys, argv)
+          yield result if block_given?
+          result
         end
       end
 

data/lib/sidekiq_unique_jobs/server.rb

@@ -5,7 +5,7 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   class Server
-    DEATH_HANDLER ||= (lambda do |job, _ex|
+    DEATH_HANDLER = (lambda do |job, _ex|
       return unless (digest = job["lock_digest"])
 
       SidekiqUniqueJobs::Digests.new.delete_by_digest(digest)
@@ -25,12 +25,25 @@ module SidekiqUniqueJobs
       config.death_handlers << death_handler
     end
 
+    #
+    # Start the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
     def self.start
       SidekiqUniqueJobs::UpdateVersion.call
       SidekiqUniqueJobs::UpgradeLocks.call
       SidekiqUniqueJobs::Orphans::Manager.start
+      SidekiqUniqueJobs::Orphans::ReaperResurrector.start
     end
 
+    #
+    # Stop the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
     def self.stop
       SidekiqUniqueJobs::Orphans::Manager.stop
     end

data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb

@@ -68,24 +68,46 @@ module Sidekiq
     prepend UniqueExtension
   end
 
-  # See Sidekiq::Api
-  class Job
-    #
-    # Provides extensions for unlocking jobs that are removed and deleted
-    #
-    # @author Mikael Henriksson <mikael@mhenrixon.com>
-    #
-    module UniqueExtension
+  if Sidekiq.const_defined?("JobRecord")
+    # See Sidekiq::Api
+    class JobRecord
       #
-      # Wraps the original method to ensure locks for the job are deleted
+      # Provides extensions for unlocking jobs that are removed and deleted
       #
-      def delete
-        SidekiqUniqueJobs::Unlockable.delete!(item)
-        super
+      # @author Mikael Henriksson <mikael@mhenrixon.com>
+      #
+      module UniqueExtension
+        #
+        # Wraps the original method to ensure locks for the job are deleted
+        #
+        def delete
+          SidekiqUniqueJobs::Unlockable.delete!(item)
+          super
+        end
       end
+
+      prepend UniqueExtension
     end
+  else
+    # See Sidekiq::Api
+    class Job
+      #
+      # Provides extensions for unlocking jobs that are removed and deleted
+      #
+      # @author Mikael Henriksson <mikael@mhenrixon.com>
+      #
+      module UniqueExtension
+        #
+        # Wraps the original method to ensure locks for the job are deleted
+        #
+        def delete
+          SidekiqUniqueJobs::Unlockable.delete!(item)
+          super
+        end
+      end
 
-    prepend UniqueExtension
+      prepend UniqueExtension
+    end
   end
 
   # See Sidekiq::Api

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -4,7 +4,7 @@
 # Contains configuration and utility methods that belongs top level
 #
 # @author Mikael Henriksson <mikael@mhenrixon.com>
-module SidekiqUniqueJobs
+module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
   include SidekiqUniqueJobs::Connection
   extend SidekiqUniqueJobs::JSON
 
@@ -188,7 +188,7 @@ module SidekiqUniqueJobs
   # @return [String] a string like `5.0.2`
   #
   def fetch_redis_version
-    redis { |conn| conn.info("server")["redis_version"] }
+    Sidekiq.redis_info["redis_version"]
   end
 
   #
@@ -236,4 +236,59 @@ module SidekiqUniqueJobs
     lock_config = validate_worker(options)
     raise InvalidWorker, lock_config unless lock_config.errors.empty?
   end
+
+  # Attempt to constantize a string worker_class argument, always
+  # failing back to the original argument when the constant can't be found
+  #
+  # @return [Sidekiq::Worker]
+  def constantize(str)
+    return str.class             if str.is_a?(Sidekiq::Worker) # sidekiq v6.x
+    return str                   unless str.is_a?(String)
+    return Object.const_get(str) unless str.include?("::")
+
+    names = str.split("::")
+    names.shift if names.empty? || names.first.empty?
+
+    names.inject(Object) do |constant, name|
+      # the false flag limits search for name to under the constant namespace
+      #   which mimics Rails' behaviour
+      constant.const_get(name, false)
+    end
+  end
+
+  # Attempt to constantize a string worker_class argument, always
+  # failing back to the original argument when the constant can't be found
+  #
+  # @return [Sidekiq::Worker, String]
+  def safe_constantize(str)
+    constantize(str)
+  rescue NameError => ex
+    case ex.message
+    when /uninitialized constant/
+      str
+    else
+      raise
+    end
+  end
+
+  #
+  # Collection with notifications
+  #
+  #
+  # @return [Reflections]
+  #
+  def reflections
+    @reflections ||= Reflections.new
+  end
+
+  #
+  # Yields notification stack for sidekiq unique jobs to configure notifications
+  #
+  #
+  # @return [void] <description>
+  #
+  # @yieldparam [Reflections] x used to configure notifications
+  def reflect
+    yield reflections if block_given?
+  end
 end

data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb

@@ -49,17 +49,7 @@ module SidekiqUniqueJobs
     #
     # @return [Sidekiq::Worker]
     def worker_class_constantize(klazz = @worker_class)
-      return klazz.class if klazz.is_a?(Sidekiq::Worker) # sidekiq v6.x
-      return klazz unless klazz.is_a?(String)
-
-      Object.const_get(klazz)
-    rescue NameError => ex
-      case ex.message
-      when /uninitialized constant/
-        klazz
-      else
-        raise
-      end
+      SidekiqUniqueJobs.safe_constantize(klazz)
     end
 
     #