sidekiq-unique-jobs 7.1.8 → 8.0.3

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -29,10 +29,10 @@ module SidekiqUniqueJobs
     # @yieldparam [<type>] if <description>
     # @yieldreturn [<type>] <describe what yield should return>
     def call(worker_class, item, queue, redis_pool = nil)
-      @worker_class = worker_class
-      @item         = item
-      @queue        = queue
-      @redis_pool   = redis_pool
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = worker_class
       return yield if unique_disabled?
 
       SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -12,33 +12,6 @@ module SidekiqUniqueJobs
       def call
         log_info { "Adding dead #{item[CLASS]} job #{item[JID]}" }
 
-        if deadset_kill?
-          deadset_kill
-        else
-          push_to_deadset
-        end
-      end
-
-      #
-      # Sidekiq version compatibility check
-      # @api private
-      #
-      #
-      # @return [true, false] depending on if Sidekiq::Deadset responds to kill
-      #
-      def deadset_kill?
-        deadset.respond_to?(:kill)
-      end
-
-      #
-      # Use Sidekiqs built in Sidekiq::DeadSet#kill
-      #   to get rid of the job
-      # @api private
-      #
-      #
-      # @return [void]
-      #
-      def deadset_kill
         if kill_with_options?
           kill_job_with_options
         else
@@ -88,22 +61,6 @@ module SidekiqUniqueJobs
         @deadset ||= Sidekiq::DeadSet.new
       end
 
-      #
-      # Used for compatibility with older Sidekiq versions
-      #
-      #
-      # @return [void]
-      #
-      def push_to_deadset
-        redis do |conn|
-          conn.multi do
-            conn.zadd("dead", now_f, payload)
-            conn.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
-            conn.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
-          end
-        end
-      end
-
       #
       # The Sidekiq job hash as JSON
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -14,14 +14,14 @@ module SidekiqUniqueJobs
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @worker_class = item[CLASS]
+        self.job_class = item[CLASS]
       end
 
       # Create a new job from the current one.
       #   This will mess up sidekiq stats because a new job is created
       def call
-        if sidekiq_worker_class?
-          if worker_class.perform_in(5, *item[ARGS])
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
             reflect(:rescheduled, item)
           else
             reflect(:reschedule_failed, item)

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
   # Requires the following methods to be defined in the including class
   #   1. item (required)
   #   2. options (can be nil)
-  #   3. worker_class (required, can be anything)
+  #   3. job_class (required, can be anything)
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
@@ -55,7 +55,7 @@ module SidekiqUniqueJobs
     # The type of lock for this worker
     #
     #
-    # @return [Symbol]
+    # @return [Symbol, NilClass]
     #
     def lock_type
       @lock_type ||= options[LOCK] || item[LOCK]
@@ -69,8 +69,8 @@ module SidekiqUniqueJobs
     #
     def options
       @options ||= begin
-        opts = default_worker_options.dup
-        opts.merge!(worker_options) if sidekiq_worker_class?
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
         (opts || {}).stringify_keys
       end
     end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -20,7 +20,7 @@ module SidekiqUniqueJobs
         call_script(
           :reap_orphans,
           conn,
-          keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
           argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "concurrent/version"
+
 module SidekiqUniqueJobs
   module Orphans
     #
@@ -70,7 +72,7 @@ module SidekiqUniqueJobs
       # @return [<type>] <description>
       #
       def task
-        @task ||= default_task
+        @task ||= default_task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
       end
 
       #
@@ -98,7 +100,7 @@ module SidekiqUniqueJobs
       # @return [void]
       #
       def task=(task)
-        @task = task
+        @task = task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
       end
 
       #
@@ -108,9 +110,7 @@ module SidekiqUniqueJobs
       # @return [Hash]
       #
       def timer_task_options
-        { run_now: true,
-          execution_interval: reaper_interval,
-          timeout_interval: reaper_timeout }
+        { run_now: true, execution_interval: reaper_interval }
       end
 
       #
@@ -127,13 +127,6 @@ module SidekiqUniqueJobs
         SidekiqUniqueJobs.config.reaper_interval
       end
 
-      #
-      # @see SidekiqUniqueJobs::Config#reaper_timeout
-      #
-      def reaper_timeout
-        SidekiqUniqueJobs.config.reaper_timeout
-      end
-
       #
       # A context to use for all log entries
       #
@@ -224,7 +217,7 @@ module SidekiqUniqueJobs
 
       #
       # Reaper interval with a little drift
-      #   Redis isn't exact enough so to give a little bufffer,
+      #   Redis isn't exact enough so to give a little buffer,
       #   we add a tiny value to the reaper interval.
       #
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -9,11 +9,17 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
+    # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
       #
+      # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
+      MAX_QUEUE_LENGTH = 1000
+      #
       # @!attribute [r] digests
       #   @return [SidekiqUniqueJobs::Digests] digest collection
       attr_reader :digests
@@ -26,6 +32,22 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -33,9 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -45,22 +70,55 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of reaped locks
       #
       def call
+        return if queues_very_full?
+
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+      end
+
       #
       # Find orphaned digests
       #
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
-          next if belongs_to_job?(digest)
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        page = 0
+        per = reaper_count * 2
+        orphans = []
+        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+
+        while results.size.positive?
+          results.each do |digest|
+            break if timeout?
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
 
-          memo << digest
-          break if memo.size >= reaper_count
+          break if timeout?
+          break if orphans.size >= reaper_count
+
+          page += 1
+          results = conn.zrange(digests.key, page * per, (page + 1) * per)
         end
+
+        orphans
+      end
+
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_source
       end
 
       #
@@ -122,18 +180,18 @@ module SidekiqUniqueJobs
 
       def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         Sidekiq.redis do |conn|
-          procs = conn.sscan_each("processes").to_a
+          procs = conn.sscan("processes").to_a
           return false if procs.empty?
 
           procs.sort.each do |key|
-            valid, workers = conn.pipelined do
+            valid, workers = conn.pipelined do |pipeline|
               # TODO: Remove the if statement in the future
-              if conn.respond_to?(:exists?)
-                conn.exists?(key)
+              if pipeline.respond_to?(:exists?)
+                pipeline.exists?(key)
               else
-                conn.exists(key)
+                pipeline.exists(key)
               end
-              conn.hgetall("#{key}:workers")
+              pipeline.hgetall("#{key}:work")
             end
 
             next unless valid
@@ -173,7 +231,7 @@ module SidekiqUniqueJobs
       # @yield queues one at a time
       #
       def queues(conn, &block)
-        conn.sscan_each("queues", &block)
+        conn.sscan("queues").each(&block)
       end
 
       def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
@@ -185,6 +243,7 @@ module SidekiqUniqueJobs
 
         loop do
           range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -194,7 +253,26 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
+        end
+      end
+
+      # If sidekiq queues are very full, it becomes highly inefficient for the reaper
+      # because it must check every queued job to verify a digest is safe to delete
+      # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
+      # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
+      def queues_very_full?
+        total_queue_size = 0
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            total_queue_size += conn.llen("queue:#{queue}")
+
+            return true if total_queue_size > MAX_QUEUE_LENGTH
+          end
         end
+        false
       end
 
       #
@@ -208,8 +286,9 @@ module SidekiqUniqueJobs
       # @return [false] when missing
       #
       def in_sorted_set?(key, digest)
-        conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
+        conn.zscan(key, match: "*#{digest}*", count: 1).to_a.any?
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -8,6 +8,13 @@ module SidekiqUniqueJobs
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     class SortedSet < Entity
+      #
+      # @return [Integer] the number of matches to return by default
+      DEFAULT_COUNT = 1_000
+      #
+      # @return [String] the default pattern to use for matching
+      SCAN_PATTERN  = "*"
+
       #
       # Return entries for this sorted set
       #
@@ -17,7 +24,7 @@ module SidekiqUniqueJobs
       # @return [Hash] when given with_scores: true
       #
       def entries(with_scores: true)
-        entrys = redis { |conn| conn.zrange(key, 0, -1, with_scores: with_scores) }
+        entrys = redis { |conn| conn.zrange(key, 0, -1, withscores: with_scores) }
         return entrys unless with_scores
 
         entrys.each_with_object({}) { |pair, hash| hash[pair[0]] = pair[1] }
@@ -33,7 +40,7 @@ module SidekiqUniqueJobs
       def add(values)
         redis do |conn|
           if values.is_a?(Array)
-            conn.zadd(key, values)
+            conn.zadd(key, *values)
           else
             conn.zadd(key, now_f, values)
           end

data/lib/sidekiq_unique_jobs/redis/string.rb

@@ -25,7 +25,9 @@ module SidekiqUniqueJobs
       #
       # @return [true, false]
       #
-      def set(obj)
+      def set(obj, pipeline = nil)
+        return pipeline.set(key, obj) if pipeline
+
         redis { |conn| conn.set(key, obj) }
       end
 

data/lib/sidekiq_unique_jobs/reflections.rb

@@ -50,7 +50,7 @@ module SidekiqUniqueJobs
     #
     # @return [void] <description>
     #
-    def dispatch(reflection, *args)
+    def dispatch(reflection, *args) # rubocop:disable Metrics/MethodLength
       if (block = @reflections[reflection])
         block.call(*args)
 

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -54,14 +54,9 @@ module SidekiqUniqueJobs
       # Only used to reduce a little bit of duplication
       # @see call_script
       def do_call(file_name, conn, keys, argv)
-        argv = argv.dup.concat([
-                                 now_f,
-                                 debug_lua,
-                                 max_history,
-                                 file_name,
-                                 redis_version,
-                               ])
-        Script.execute(file_name, conn, keys: keys, argv: argv)
+        argv = argv.dup.push(now_f, debug_lua, max_history, file_name, redis_version)
+
+        Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
       end
 
       #
@@ -122,6 +117,17 @@ module SidekiqUniqueJobs
       def redis_version
         SidekiqUniqueJobs.config.redis_version
       end
+
+      def normalize_argv(argv)
+        argv.each_with_index do |item, index|
+          case item
+          when FalseClass, NilClass
+            argv[index] = 0
+          when TrueClass
+            argv[index] = 1
+          end
+        end
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/server.rb

@@ -33,7 +33,6 @@ module SidekiqUniqueJobs
     #
     def self.start
       SidekiqUniqueJobs::UpdateVersion.call
-      SidekiqUniqueJobs::UpgradeLocks.call
       SidekiqUniqueJobs::Orphans::Manager.start
       SidekiqUniqueJobs::Orphans::ReaperResurrector.start
     end

data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb

@@ -68,7 +68,7 @@ module Sidekiq
     prepend UniqueExtension
   end
 
-  if Sidekiq.const_defined?("JobRecord")
+  if Sidekiq.const_defined?(:JobRecord)
     # See Sidekiq::Api
     class JobRecord
       #

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -17,7 +17,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
   # @return [SidekiqUniqueJobs::Config] the gem configuration
   #
   def config
-    @config ||= reset!
+    @config ||= reset! # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
   end
 
   #
@@ -71,6 +71,16 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
     config.logger = other
   end
 
+  #
+  # Check if logging is enabled
+  #
+  #
+  # @return [true, false]
+  #
+  def logging?
+    config.logger_enabled
+  end
+
   #
   # Temporarily use another configuration and reset to the old config after yielding
   #
@@ -98,7 +108,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
   # @return [SidekiqUniqueJobs::Config] a default gem configuration
   #
   def reset!
-    @config = SidekiqUniqueJobs::Config.default
+    @config = SidekiqUniqueJobs::Config.default # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
   end
 
   #
@@ -188,7 +198,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
   # @return [String] a string like `5.0.2`
   #
   def fetch_redis_version
-    Sidekiq.redis_info["redis_version"]
+    Sidekiq.default_configuration.redis_info["redis_version"]
   end
 
   #
@@ -278,7 +288,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
   # @return [Reflections]
   #
   def reflections
-    @reflections ||= Reflections.new
+    @reflections ||= Reflections.new # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
   end
 
   #

data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb

@@ -5,41 +5,56 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module SidekiqWorkerMethods
+    #
+    # @!attribute [r] job_class
+    #   @return [Sidekiq::Worker] The Sidekiq::Worker implementation
+    attr_reader :job_class
+
     # Avoids duplicating worker_class.respond_to? in multiple places
     # @return [true, false]
-    def worker_method_defined?(method_sym)
-      worker_class.respond_to?(method_sym)
+    def job_method_defined?(method_sym)
+      job_class.respond_to?(method_sym)
     end
 
     # Wraps #get_sidekiq_options to always work with a hash
     # @return [Hash] of the worker class sidekiq options
-    def worker_options
-      return {} unless sidekiq_worker_class?
+    def job_options
+      return {} unless sidekiq_job_class?
 
-      worker_class.get_sidekiq_options.deep_stringify_keys
+      job_class.get_sidekiq_options.deep_stringify_keys
     end
 
     # Tests that the
-    # @return [true] if worker_class responds to get_sidekiq_options
-    # @return [false] if worker_class does not respond to get_sidekiq_options
-    def sidekiq_worker_class?
-      worker_method_defined?(:get_sidekiq_options)
+    # @return [true] if job_class responds to get_sidekiq_options
+    # @return [false] if job_class does not respond to get_sidekiq_options
+    def sidekiq_job_class?
+      job_method_defined?(:get_sidekiq_options)
     end
 
-    # The Sidekiq::Worker implementation
-    # @return [Sidekiq::Worker]
-    def worker_class
-      @_worker_class ||= worker_class_constantize # rubocop:disable Naming/MemoizedInstanceVariableName
+    def job_class=(obj)
+      # this is what was originally passed in, it can be an instance or a class depending on sidekiq version
+      @original_job_class = obj
+      @job_class = job_class_constantize(obj)
     end
 
     # The hook to call after a successful unlock
     # @return [Proc]
-    def after_unlock_hook
+    def after_unlock_hook # rubocop:disable Metrics/MethodLength
       lambda do
-        if @worker_class.respond_to?(:after_unlock)
-          @worker_class.after_unlock # instance method in sidekiq v6
-        elsif worker_class.respond_to?(:after_unlock)
-          worker_class.after_unlock # class method regardless of sidekiq version
+        if @original_job_class.respond_to?(:after_unlock)
+          # instance method in sidekiq v6
+          if @original_job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
+            @original_job_class.after_unlock(item)
+          else
+            @original_job_class.after_unlock
+          end
+        elsif job_class.respond_to?(:after_unlock)
+          # class method regardless of sidekiq version
+          if job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
+            job_class.after_unlock(item)
+          else
+            job_class.after_unlock
+          end
         end
       end
     end
@@ -48,7 +63,7 @@ module SidekiqUniqueJobs
     # failing back to the original argument when the constant can't be found
     #
     # @return [Sidekiq::Worker]
-    def worker_class_constantize(klazz = @worker_class)
+    def job_class_constantize(klazz = @job_class)
       SidekiqUniqueJobs.safe_constantize(klazz)
     end
 
@@ -58,8 +73,12 @@ module SidekiqUniqueJobs
     #
     # @return [Hash<Symbol, Object>]
     #
-    def default_worker_options
-      Sidekiq.default_worker_options
+    def default_job_options
+      if Sidekiq.respond_to?(:default_job_options)
+        Sidekiq.default_job_options
+      else
+        Sidekiq.default_worker_options
+      end
     end
   end
 end