sidekiq-unique-jobs 7.1.5 → 7.1.25

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -76,6 +76,9 @@ if pttl and pttl > 0 then
 
   log_debug("PEXPIRE", locked, pttl)
   redis.call("PEXPIRE", locked, pttl)
+
+  log_debug("PEXPIRE", info, pttl)
+  redis.call("PEXPIRE", info, pttl)
 end
 
 log_debug("PEXPIRE", queued, 1000)
@@ -84,9 +87,6 @@ redis.call("PEXPIRE", queued, 1000)
 log_debug("PEXPIRE", primed, 1000)
 redis.call("PEXPIRE", primed, 1000)
 
-log_debug("PEXPIRE", info, 1000)
-redis.call("PEXPIRE", info, 1000)
-
 log("Locked")
 log_debug("END lock digest:", digest, "job_id:", job_id)
 return job_id

data/lib/sidekiq_unique_jobs/lua/shared/_common.lua

@@ -31,7 +31,7 @@ local function log(message, prev_jid)
   log_debug("ZADD", changelog, current_time, entry);
   redis.call("ZADD", changelog, current_time, entry);
   local total_entries = redis.call("ZCARD", changelog)
-  local removed_entries = redis.call("ZREMRANGEBYRANK", changelog, max_history, -1)
+  local removed_entries = redis.call("ZREMRANGEBYRANK", changelog, 0, -1 * max_history)
   if removed_entries > 0 then
     log_debug("Removing", removed_entries , "entries from changelog (total entries", total_entries, "exceeds max_history:", max_history ..")");
   end

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -65,11 +65,6 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, -1, job_id)
 redis.call("LREM", primed, -1, job_id)
 
-if limit and limit <= 1 and locked_count and locked_count <= 1 then
-  log_debug("ZREM", digests, digest)
-  redis.call("ZREM", digests, digest)
-end
-
 local redis_version = toversion(redisversion)
 local del_cmd       = "DEL"
 
@@ -83,6 +78,18 @@ if lock_type ~= "until_expired" then
   redis.call("HDEL", locked, job_id)
 end
 
+local locked_count = redis.call("HLEN", locked)
+
+if locked_count and locked_count < 1 then
+  log_debug(del_cmd, locked)
+  redis.call(del_cmd, locked)
+end
+
+if limit and limit <= 1 and locked_count and locked_count <= 1 then
+  log_debug("ZREM", digests, digest)
+  redis.call("ZREM", digests, digest)
+end
+
 log_debug("LPUSH", queued, "1")
 redis.call("LPUSH", queued, "1")
 

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
+      include Sidekiq::ClientMiddleware if defined?(Sidekiq::ClientMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware
@@ -30,7 +32,7 @@ module SidekiqUniqueJobs
       private
 
       def lock
-        lock_instance.lock do |_locked_jid|
+        lock_instance.lock do
           reflect(:locked, item)
           return yield
         end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -29,10 +29,10 @@ module SidekiqUniqueJobs
     # @yieldparam [<type>] if <description>
     # @yieldreturn [<type>] <describe what yield should return>
     def call(worker_class, item, queue, redis_pool = nil)
-      @worker_class = worker_class
-      @item         = item
-      @queue        = queue
-      @redis_pool   = redis_pool
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = item[CLASS]
       return yield if unique_disabled?
 
       SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -96,10 +96,10 @@ module SidekiqUniqueJobs
       #
       def push_to_deadset
         redis do |conn|
-          conn.multi do
-            conn.zadd("dead", now_f, payload)
-            conn.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
-            conn.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
+          conn.multi do |pipeline|
+            pipeline.zadd("dead", now_f, payload)
+            pipeline.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
+            pipeline.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
           end
         end
       end

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -14,14 +14,14 @@ module SidekiqUniqueJobs
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @worker_class = item[CLASS]
+        self.job_class = item[CLASS]
       end
 
       # Create a new job from the current one.
       #   This will mess up sidekiq stats because a new job is created
       def call
-        if sidekiq_worker_class?
-          if worker_class.perform_in(5, *item[ARGS])
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
             reflect(:rescheduled, item)
           else
             reflect(:reschedule_failed, item)

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
   # Requires the following methods to be defined in the including class
   #   1. item (required)
   #   2. options (can be nil)
-  #   3. worker_class (required, can be anything)
+  #   3. job_class (required, can be anything)
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
@@ -46,10 +46,8 @@ module SidekiqUniqueJobs
     # @return [Class]
     #
     def lock_class
-      @lock_class ||= begin
-        locks.fetch(lock_type.to_sym) do
-          raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
-        end
+      @lock_class ||= locks.fetch(lock_type.to_sym) do
+        raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
       end
     end
 
@@ -71,8 +69,8 @@ module SidekiqUniqueJobs
     #
     def options
       @options ||= begin
-        opts = default_worker_options.dup
-        opts.merge!(worker_options) if sidekiq_worker_class?
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
         (opts || {}).stringify_keys
       end
     end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "concurrent/version"
+
 module SidekiqUniqueJobs
   module Orphans
     #
@@ -10,10 +12,18 @@ module SidekiqUniqueJobs
     module Manager
       module_function
 
+      #
+      # @return [Float] the amount to add to the reaper interval
       DRIFT_FACTOR = 0.02
+      #
+      # @return [Symbol] allowed reapers (:ruby or :lua)
       REAPERS      = [:ruby, :lua].freeze
 
+      # includes "SidekiqUniqueJobs::Connection"
+      # @!parse include SidekiqUniqueJobs::Connection
       include SidekiqUniqueJobs::Connection
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::Logging
 
       #
@@ -65,6 +75,12 @@ module SidekiqUniqueJobs
         @task ||= default_task
       end
 
+      #
+      # A properly configured timer task
+      #
+      #
+      # @return [SidekiqUniqueJobs::TimerTask]
+      #
       def default_task
         SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
           with_logging_context do
@@ -76,6 +92,13 @@ module SidekiqUniqueJobs
         end
       end
 
+      #
+      # Store a task to use for scheduled execution
+      #
+      # @param [SidekiqUniqueJobs::TimerTask] task the task to use
+      #
+      # @return [void]
+      #
       def task=(task)
         @task = task
       end
@@ -87,9 +110,7 @@ module SidekiqUniqueJobs
       # @return [Hash]
       #
       def timer_task_options
-        { run_now: true,
-          execution_interval: reaper_interval,
-          timeout_interval: reaper_timeout }
+        { run_now: true, execution_interval: reaper_interval }
       end
 
       #
@@ -106,13 +127,6 @@ module SidekiqUniqueJobs
         SidekiqUniqueJobs.config.reaper_interval
       end
 
-      #
-      # @see SidekiqUniqueJobs::Config#reaper_timeout
-      #
-      def reaper_timeout
-        SidekiqUniqueJobs.config.reaper_timeout
-      end
-
       #
       # A context to use for all log entries
       #
@@ -201,10 +215,24 @@ module SidekiqUniqueJobs
         redis { |conn| conn.del(UNIQUE_REAPER) }
       end
 
+      #
+      # Reaper interval with a little drift
+      #   Redis isn't exact enough so to give a little bufffer,
+      #   we add a tiny value to the reaper interval.
+      #
+      #
+      # @return [Integer] <description>
+      #
       def drift_reaper_interval
         reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
       end
 
+      #
+      # Current time (as integer value)
+      #
+      #
+      # @return [Integer]
+      #
       def current_timestamp
         Time.now.to_i
       end

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -9,9 +9,17 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
+    # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
+      #
+      # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
       #
+      # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
+      MAX_QUEUE_LENGTH = 1000
+      #
       # @!attribute [r] digests
       #   @return [SidekiqUniqueJobs::Digests] digest collection
       attr_reader :digests
@@ -24,6 +32,16 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as integer
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -31,9 +49,11 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests    = SidekiqUniqueJobs::Digests.new
+        @scheduled  = Redis::SortedSet.new(SCHEDULE)
+        @retried    = Redis::SortedSet.new(RETRY)
+        @start_time = time_source.call
+        @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -43,6 +63,8 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of reaped locks
       #
       def call
+        return if queues_very_full?
+
         BatchDelete.call(orphans, conn)
       end
 
@@ -52,13 +74,37 @@ module SidekiqUniqueJobs
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
-          next if belongs_to_job?(digest)
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        page = 0
+        per = reaper_count * 2
+        orphans = []
+        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+
+        while results.size.positive?
+          results.each do |digest|
+            break if timeout?
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
+
+          break if timeout?
+          break if orphans.size >= reaper_count
 
-          memo << digest
-          break if memo.size >= reaper_count
+          page += 1
+          results = conn.zrange(digests.key, page * per, (page + 1) * per)
         end
+
+        orphans
+      end
+
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_time
       end
 
       #
@@ -124,14 +170,14 @@ module SidekiqUniqueJobs
           return false if procs.empty?
 
           procs.sort.each do |key|
-            valid, workers = conn.pipelined do
+            valid, workers = conn.pipelined do |pipeline|
               # TODO: Remove the if statement in the future
-              if conn.respond_to?(:exists?)
-                conn.exists?(key)
+              if pipeline.respond_to?(:exists?)
+                pipeline.exists?(key)
               else
-                conn.exists(key)
+                pipeline.exists(key)
               end
-              conn.hgetall("#{key}:workers")
+              pipeline.hgetall("#{key}:workers")
             end
 
             next unless valid
@@ -182,7 +228,7 @@ module SidekiqUniqueJobs
         page_size    = 50
 
         loop do
-          range_start = page * page_size - deleted_size
+          range_start = (page * page_size) - deleted_size
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -195,6 +241,22 @@ module SidekiqUniqueJobs
         end
       end
 
+      # If sidekiq queues are very full, it becomes highly inefficient for the reaper
+      # because it must check every queued job to verify a digest is safe to delete
+      # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
+      # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
+      def queues_very_full?
+        total_queue_size = 0
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            total_queue_size += conn.llen("queue:#{queue}")
+
+            return true if total_queue_size > MAX_QUEUE_LENGTH
+          end
+        end
+        false
+      end
+
       #
       # Checks a sorted set for the existance of this digest
       #
@@ -209,5 +271,6 @@ module SidekiqUniqueJobs
         conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/sidekiq_unique_jobs/redis/string.rb

@@ -25,7 +25,9 @@ module SidekiqUniqueJobs
       #
       # @return [true, false]
       #
-      def set(obj)
+      def set(obj, pipeline = nil)
+        return pipeline.set(key, obj) if pipeline
+
         redis { |conn| conn.set(key, obj) }
       end
 

data/lib/sidekiq_unique_jobs/reflectable.rb

@@ -7,11 +7,20 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   module Reflectable
-    def reflect(name, *args)
-      SidekiqUniqueJobs.reflections.dispatch(name, *args)
+    #
+    # Reflects on specific event
+    #
+    # @param [Symbol] reflection the reflected event
+    # @param [Array] args arguments to provide to reflector
+    #
+    # @return [void]
+    #
+    def reflect(reflection, *args)
+      SidekiqUniqueJobs.reflections.dispatch(reflection, *args)
       nil
     rescue UniqueJobsError => ex
       SidekiqUniqueJobs.logger.error(ex)
+      nil
     end
   end
 end

data/lib/sidekiq_unique_jobs/reflections.rb

@@ -42,6 +42,14 @@ module SidekiqUniqueJobs
       @reflections = {}
     end
 
+    #
+    # Dispatch a reflected event
+    #
+    # @param [reflection] reflection the reflected event
+    # @param [Array] args the arguments to provide to the block
+    #
+    # @return [void] <description>
+    #
     def dispatch(reflection, *args)
       if (block = @reflections[reflection])
         block.call(*args)
@@ -49,12 +57,15 @@ module SidekiqUniqueJobs
         if DEPRECATIONS.key?(reflection)
           replacement, removal_version = DEPRECATIONS[reflection]
           SidekiqUniqueJobs::Deprecation.warn(
-            "#{reflection} is deprecated and will be removed in version #{removal_version}. Use #{replacement} instead.",
+            "#{reflection} is deprecated and will be removed in version #{removal_version}." \
+            " Use #{replacement} instead.",
           )
         end
       elsif misconfigured?(reflection)
         raise NoSuchNotificationError, reflection
       end
+
+      nil
     end
 
     def configured?(reflection)

data/lib/sidekiq_unique_jobs/server.rb

@@ -5,7 +5,7 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   class Server
-    DEATH_HANDLER ||= (lambda do |job, _ex|
+    DEATH_HANDLER = (lambda do |job, _ex|
       return unless (digest = job["lock_digest"])
 
       SidekiqUniqueJobs::Digests.new.delete_by_digest(digest)
@@ -25,6 +25,12 @@ module SidekiqUniqueJobs
       config.death_handlers << death_handler
     end
 
+    #
+    # Start the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
     def self.start
       SidekiqUniqueJobs::UpdateVersion.call
       SidekiqUniqueJobs::UpgradeLocks.call
@@ -32,6 +38,12 @@ module SidekiqUniqueJobs
       SidekiqUniqueJobs::Orphans::ReaperResurrector.start
     end
 
+    #
+    # Stop the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
     def self.stop
       SidekiqUniqueJobs::Orphans::Manager.stop
     end

data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb

@@ -68,24 +68,46 @@ module Sidekiq
     prepend UniqueExtension
   end
 
-  # See Sidekiq::Api
-  class Job
-    #
-    # Provides extensions for unlocking jobs that are removed and deleted
-    #
-    # @author Mikael Henriksson <mikael@mhenrixon.com>
-    #
-    module UniqueExtension
+  if Sidekiq.const_defined?(:JobRecord)
+    # See Sidekiq::Api
+    class JobRecord
       #
-      # Wraps the original method to ensure locks for the job are deleted
+      # Provides extensions for unlocking jobs that are removed and deleted
       #
-      def delete
-        SidekiqUniqueJobs::Unlockable.delete!(item)
-        super
+      # @author Mikael Henriksson <mikael@mhenrixon.com>
+      #
+      module UniqueExtension
+        #
+        # Wraps the original method to ensure locks for the job are deleted
+        #
+        def delete
+          SidekiqUniqueJobs::Unlockable.delete!(item)
+          super
+        end
       end
+
+      prepend UniqueExtension
     end
+  else
+    # See Sidekiq::Api
+    class Job
+      #
+      # Provides extensions for unlocking jobs that are removed and deleted
+      #
+      # @author Mikael Henriksson <mikael@mhenrixon.com>
+      #
+      module UniqueExtension
+        #
+        # Wraps the original method to ensure locks for the job are deleted
+        #
+        def delete
+          SidekiqUniqueJobs::Unlockable.delete!(item)
+          super
+        end
+      end
 
-    prepend UniqueExtension
+      prepend UniqueExtension
+    end
   end
 
   # See Sidekiq::Api