sidekiq-unique-jobs 7.1.8 → 8.0.2

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -9,11 +9,17 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
+    # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
       #
+      # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
+      MAX_QUEUE_LENGTH = 1000
+      #
       # @!attribute [r] digests
       #   @return [SidekiqUniqueJobs::Digests] digest collection
       attr_reader :digests
@@ -26,6 +32,22 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -33,9 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -45,22 +70,55 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of reaped locks
       #
       def call
+        return if queues_very_full?
+
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+      end
+
       #
       # Find orphaned digests
       #
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
-          next if belongs_to_job?(digest)
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        page = 0
+        per = reaper_count * 2
+        orphans = []
+        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+
+        while results.size.positive?
+          results.each do |digest|
+            break if timeout?
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
 
-          memo << digest
-          break if memo.size >= reaper_count
+          break if timeout?
+          break if orphans.size >= reaper_count
+
+          page += 1
+          results = conn.zrange(digests.key, page * per, (page + 1) * per)
         end
+
+        orphans
+      end
+
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_source
       end
 
       #
@@ -122,18 +180,18 @@ module SidekiqUniqueJobs
 
       def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         Sidekiq.redis do |conn|
-          procs = conn.sscan_each("processes").to_a
+          procs = conn.sscan("processes").to_a
           return false if procs.empty?
 
           procs.sort.each do |key|
-            valid, workers = conn.pipelined do
+            valid, workers = conn.pipelined do |pipeline|
               # TODO: Remove the if statement in the future
-              if conn.respond_to?(:exists?)
-                conn.exists?(key)
+              if pipeline.respond_to?(:exists?)
+                pipeline.exists?(key)
               else
-                conn.exists(key)
+                pipeline.exists(key)
               end
-              conn.hgetall("#{key}:workers")
+              pipeline.hgetall("#{key}:work")
             end
 
             next unless valid
@@ -173,7 +231,7 @@ module SidekiqUniqueJobs
       # @yield queues one at a time
       #
       def queues(conn, &block)
-        conn.sscan_each("queues", &block)
+        conn.sscan("queues").each(&block)
       end
 
       def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
@@ -185,6 +243,7 @@ module SidekiqUniqueJobs
 
         loop do
           range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -194,7 +253,26 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
+        end
+      end
+
+      # If sidekiq queues are very full, it becomes highly inefficient for the reaper
+      # because it must check every queued job to verify a digest is safe to delete
+      # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
+      # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
+      def queues_very_full?
+        total_queue_size = 0
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            total_queue_size += conn.llen("queue:#{queue}")
+
+            return true if total_queue_size > MAX_QUEUE_LENGTH
+          end
         end
+        false
       end
 
       #
@@ -208,8 +286,9 @@ module SidekiqUniqueJobs
       # @return [false] when missing
       #
       def in_sorted_set?(key, digest)
-        conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
+        conn.zscan(key, match: "*#{digest}*", count: 1).to_a.any?
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -8,6 +8,13 @@ module SidekiqUniqueJobs
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     class SortedSet < Entity
+      #
+      # @return [Integer] the number of matches to return by default
+      DEFAULT_COUNT = 1_000
+      #
+      # @return [String] the default pattern to use for matching
+      SCAN_PATTERN  = "*"
+
       #
       # Return entries for this sorted set
       #
@@ -17,7 +24,7 @@ module SidekiqUniqueJobs
       # @return [Hash] when given with_scores: true
       #
       def entries(with_scores: true)
-        entrys = redis { |conn| conn.zrange(key, 0, -1, with_scores: with_scores) }
+        entrys = redis { |conn| conn.zrange(key, 0, -1, withscores: with_scores) }
         return entrys unless with_scores
 
         entrys.each_with_object({}) { |pair, hash| hash[pair[0]] = pair[1] }
@@ -33,7 +40,7 @@ module SidekiqUniqueJobs
       def add(values)
         redis do |conn|
           if values.is_a?(Array)
-            conn.zadd(key, values)
+            conn.zadd(key, *values)
           else
             conn.zadd(key, now_f, values)
           end

data/lib/sidekiq_unique_jobs/redis/string.rb

@@ -25,7 +25,9 @@ module SidekiqUniqueJobs
       #
       # @return [true, false]
       #
-      def set(obj)
+      def set(obj, pipeline = nil)
+        return pipeline.set(key, obj) if pipeline
+
         redis { |conn| conn.set(key, obj) }
       end
 

data/lib/sidekiq_unique_jobs/reflections.rb

@@ -50,7 +50,7 @@ module SidekiqUniqueJobs
     #
     # @return [void] <description>
     #
-    def dispatch(reflection, *args)
+    def dispatch(reflection, *args) # rubocop:disable Metrics/MethodLength
       if (block = @reflections[reflection])
         block.call(*args)
 

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -54,14 +54,9 @@ module SidekiqUniqueJobs
       # Only used to reduce a little bit of duplication
       # @see call_script
       def do_call(file_name, conn, keys, argv)
-        argv = argv.dup.concat([
-                                 now_f,
-                                 debug_lua,
-                                 max_history,
-                                 file_name,
-                                 redis_version,
-                               ])
-        Script.execute(file_name, conn, keys: keys, argv: argv)
+        argv = argv.dup.push(now_f, debug_lua, max_history, file_name, redis_version)
+
+        Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
       end
 
       #
@@ -122,6 +117,17 @@ module SidekiqUniqueJobs
       def redis_version
         SidekiqUniqueJobs.config.redis_version
       end
+
+      def normalize_argv(argv)
+        argv.each_with_index do |item, index|
+          case item
+          when FalseClass, NilClass
+            argv[index] = 0
+          when TrueClass
+            argv[index] = 1
+          end
+        end
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/server.rb

@@ -33,7 +33,6 @@ module SidekiqUniqueJobs
     #
     def self.start
       SidekiqUniqueJobs::UpdateVersion.call
-      SidekiqUniqueJobs::UpgradeLocks.call
       SidekiqUniqueJobs::Orphans::Manager.start
       SidekiqUniqueJobs::Orphans::ReaperResurrector.start
     end

data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb

@@ -68,7 +68,7 @@ module Sidekiq
     prepend UniqueExtension
   end
 
-  if Sidekiq.const_defined?("JobRecord")
+  if Sidekiq.const_defined?(:JobRecord)
     # See Sidekiq::Api
     class JobRecord
       #

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -71,6 +71,16 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
     config.logger = other
   end
 
+  #
+  # Check if logging is enabled
+  #
+  #
+  # @return [true, false]
+  #
+  def logging?
+    config.logger_enabled
+  end
+
   #
   # Temporarily use another configuration and reset to the old config after yielding
   #
@@ -188,7 +198,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
   # @return [String] a string like `5.0.2`
   #
   def fetch_redis_version
-    Sidekiq.redis_info["redis_version"]
+    Sidekiq.default_configuration.redis_info["redis_version"]
   end
 
   #

data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb

@@ -5,41 +5,56 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module SidekiqWorkerMethods
+    #
+    # @!attribute [r] job_class
+    #   @return [Sidekiq::Worker] The Sidekiq::Worker implementation
+    attr_reader :job_class
+
     # Avoids duplicating worker_class.respond_to? in multiple places
     # @return [true, false]
-    def worker_method_defined?(method_sym)
-      worker_class.respond_to?(method_sym)
+    def job_method_defined?(method_sym)
+      job_class.respond_to?(method_sym)
     end
 
     # Wraps #get_sidekiq_options to always work with a hash
     # @return [Hash] of the worker class sidekiq options
-    def worker_options
-      return {} unless sidekiq_worker_class?
+    def job_options
+      return {} unless sidekiq_job_class?
 
-      worker_class.get_sidekiq_options.deep_stringify_keys
+      job_class.get_sidekiq_options.deep_stringify_keys
     end
 
     # Tests that the
-    # @return [true] if worker_class responds to get_sidekiq_options
-    # @return [false] if worker_class does not respond to get_sidekiq_options
-    def sidekiq_worker_class?
-      worker_method_defined?(:get_sidekiq_options)
+    # @return [true] if job_class responds to get_sidekiq_options
+    # @return [false] if job_class does not respond to get_sidekiq_options
+    def sidekiq_job_class?
+      job_method_defined?(:get_sidekiq_options)
     end
 
-    # The Sidekiq::Worker implementation
-    # @return [Sidekiq::Worker]
-    def worker_class
-      @_worker_class ||= worker_class_constantize # rubocop:disable Naming/MemoizedInstanceVariableName
+    def job_class=(obj)
+      # this is what was originally passed in, it can be an instance or a class depending on sidekiq version
+      @original_job_class = obj
+      @job_class = job_class_constantize(obj)
     end
 
     # The hook to call after a successful unlock
     # @return [Proc]
-    def after_unlock_hook
+    def after_unlock_hook # rubocop:disable Metrics/MethodLength
       lambda do
-        if @worker_class.respond_to?(:after_unlock)
-          @worker_class.after_unlock # instance method in sidekiq v6
-        elsif worker_class.respond_to?(:after_unlock)
-          worker_class.after_unlock # class method regardless of sidekiq version
+        if @original_job_class.respond_to?(:after_unlock)
+          # instance method in sidekiq v6
+          if @original_job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
+            @original_job_class.after_unlock(item)
+          else
+            @original_job_class.after_unlock
+          end
+        elsif job_class.respond_to?(:after_unlock)
+          # class method regardless of sidekiq version
+          if job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
+            job_class.after_unlock(item)
+          else
+            job_class.after_unlock
+          end
         end
       end
     end
@@ -48,7 +63,7 @@ module SidekiqUniqueJobs
     # failing back to the original argument when the constant can't be found
     #
     # @return [Sidekiq::Worker]
-    def worker_class_constantize(klazz = @worker_class)
+    def job_class_constantize(klazz = @job_class)
       SidekiqUniqueJobs.safe_constantize(klazz)
     end
 
@@ -58,8 +73,12 @@ module SidekiqUniqueJobs
     #
     # @return [Hash<Symbol, Object>]
     #
-    def default_worker_options
-      Sidekiq.default_worker_options
+    def default_job_options
+      if Sidekiq.respond_to?(:default_job_options)
+        Sidekiq.default_job_options
+      else
+        Sidekiq.default_worker_options
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/testing.rb

@@ -21,16 +21,24 @@ module Sidekiq
   #
   # @param [Hash<Symbol, Object>] tmp_config the temporary config to use
   #
-  def self.use_options(tmp_config = {})
-    old_options = default_worker_options.dup
+  def self.use_options(tmp_config = {}) # rubocop:disable Metrics/MethodLength
+    if respond_to?(:default_job_options)
+      default_job_options.clear
+      self.default_job_options = tmp_config
+    else
+      default_worker_options.clear
+      self.default_worker_options = tmp_config
+    end
 
-    default_worker_options.clear
-    self.default_worker_options = tmp_config
     yield
   ensure
-    default_worker_options.clear
-    self.default_worker_options = DEFAULT_WORKER_OPTIONS
-    self.default_worker_options = old_options
+    if respond_to?(:default_job_options)
+      default_job_options.clear
+      self.default_job_options = default_job_options
+    else
+      default_worker_options.clear
+      self.default_worker_options = DEFAULT_WORKER_OPTIONS
+    end
   end
 
   #
@@ -54,20 +62,14 @@ module Sidekiq
 
         yield
       ensure
-        self.sidekiq_options_hash = Sidekiq::DEFAULT_WORKER_OPTIONS
-        sidekiq_options(old_options)
-      end
+        self.sidekiq_options_hash =
+          if Sidekiq.respond_to?(:default_job_options)
+            Sidekiq.default_job_options
+          else
+            DEFAULT_WORKER_OPTIONS
+          end
 
-      #
-      # Clears the jobs for this worker and removes all locks
-      #
-      def clear
-        jobs.each do |job|
-          SidekiqUniqueJobs::Unlockable.unlock(job)
-        end
-
-        Sidekiq::Queues[queue].clear
-        jobs.clear
+        sidekiq_options(old_options)
       end
     end
 
@@ -88,6 +90,36 @@ module Sidekiq
         super(options)
       end
 
+      #
+      # Prepends deletion of locks to clear
+      #
+      module ClassMethods
+        #
+        # Clears the jobs for this worker and removes all locks
+        #
+        def clear
+          jobs.each do |job|
+            SidekiqUniqueJobs::Unlockable.unlock(job)
+          end
+
+          super
+        end
+      end
+    end
+
+    prepend Overrides
+
+    #
+    # Prepends methods to Sidekiq::Worker
+    #
+    module ClassMethods
+      prepend Overrides::ClassMethods
+    end
+
+    #
+    # Prepends singleton methods to Sidekiq::Worker
+    #
+    module SignletonOverrides
       #
       # Clears all jobs for this worker and removes all locks
       #
@@ -98,6 +130,6 @@ module Sidekiq
       end
     end
 
-    prepend Overrides
+    singleton_class.prepend SignletonOverrides
   end
 end