sidekiq-unique-jobs 7.1.5 → 7.1.30

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -1,9 +1,10 @@
 redis.replicate_commands()
 
 -------- BEGIN keys ---------
-local digests_set  = KEYS[1]
-local schedule_set = KEYS[2]
-local retry_set    = KEYS[3]
+local digests_set          = KEYS[1]
+local expiring_digests_set = KEYS[2]
+local schedule_set         = KEYS[3]
+local retry_set            = KEYS[4]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
@@ -90,5 +91,32 @@ repeat
   index = index + per
 until index >= total or del_count >= reaper_count
 
+if del_count < reaper_count then
+  index = 0
+  total = redis.call("ZCOUNT", expiring_digests_set, 0, current_time)
+  repeat
+    local digests = redis.call("ZRANGEBYSCORE", expiring_digests_set, 0, current_time, "LIMIT", index, index + per -1)
+
+    for _, digest in pairs(digests) do
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", expiring_digests_set, digest)
+      del_count = del_count + 1
+    end
+
+    index = index + per
+  until index >= total or del_count >= reaper_count
+end
+
 log_debug("END")
 return del_count

data/lib/sidekiq_unique_jobs/lua/shared/_common.lua

@@ -31,7 +31,7 @@ local function log(message, prev_jid)
   log_debug("ZADD", changelog, current_time, entry);
   redis.call("ZADD", changelog, current_time, entry);
   local total_entries = redis.call("ZCARD", changelog)
-  local removed_entries = redis.call("ZREMRANGEBYRANK", changelog, max_history, -1)
+  local removed_entries = redis.call("ZREMRANGEBYRANK", changelog, 0, -1 * max_history)
   if removed_entries > 0 then
     log_debug("Removing", removed_entries , "entries from changelog (total entries", total_entries, "exceeds max_history:", max_history ..")");
   end

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -15,7 +15,7 @@ local function find_digest_in_process_set(digest, threshold)
     log_debug("Found number of processes:", #processes, "next cursor:", next_process_cursor)
 
     for _, process in ipairs(processes) do
-      local workers_key = process .. ":workers"
+      local workers_key = process .. ":work"
       log_debug("searching in process set:", process,
                 "for digest:", digest,
                 "cursor:", process_cursor)

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -65,11 +65,6 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, -1, job_id)
 redis.call("LREM", primed, -1, job_id)
 
-if limit and limit <= 1 and locked_count and locked_count <= 1 then
-  log_debug("ZREM", digests, digest)
-  redis.call("ZREM", digests, digest)
-end
-
 local redis_version = toversion(redisversion)
 local del_cmd       = "DEL"
 
@@ -83,6 +78,23 @@ if lock_type ~= "until_expired" then
   redis.call("HDEL", locked, job_id)
 end
 
+local locked_count = redis.call("HLEN", locked)
+
+if locked_count and locked_count < 1 then
+  log_debug(del_cmd, locked)
+  redis.call(del_cmd, locked)
+end
+
+if redis.call("LLEN", primed) == 0 then
+  log_debug(del_cmd, primed)
+  redis.call(del_cmd, primed)
+end
+
+if limit and limit <= 1 and locked_count and locked_count <= 1 then
+  log_debug("ZREM", digests, digest)
+  redis.call("ZREM", digests, digest)
+end
+
 log_debug("LPUSH", queued, "1")
 redis.call("LPUSH", queued, "1")
 

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
+      include Sidekiq::ClientMiddleware if defined?(Sidekiq::ClientMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware
@@ -30,7 +32,7 @@ module SidekiqUniqueJobs
       private
 
       def lock
-        lock_instance.lock do |_locked_jid|
+        lock_instance.lock do
           reflect(:locked, item)
           return yield
         end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -29,10 +29,10 @@ module SidekiqUniqueJobs
     # @yieldparam [<type>] if <description>
     # @yieldreturn [<type>] <describe what yield should return>
     def call(worker_class, item, queue, redis_pool = nil)
-      @worker_class = worker_class
-      @item         = item
-      @queue        = queue
-      @redis_pool   = redis_pool
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = worker_class
       return yield if unique_disabled?
 
       SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -96,10 +96,10 @@ module SidekiqUniqueJobs
       #
       def push_to_deadset
         redis do |conn|
-          conn.multi do
-            conn.zadd("dead", now_f, payload)
-            conn.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
-            conn.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
+          conn.multi do |pipeline|
+            pipeline.zadd("dead", now_f, payload)
+            pipeline.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
+            pipeline.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
           end
         end
       end

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -14,14 +14,14 @@ module SidekiqUniqueJobs
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @worker_class = item[CLASS]
+        self.job_class = item[CLASS]
       end
 
       # Create a new job from the current one.
       #   This will mess up sidekiq stats because a new job is created
       def call
-        if sidekiq_worker_class?
-          if worker_class.perform_in(5, *item[ARGS])
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
             reflect(:rescheduled, item)
           else
             reflect(:reschedule_failed, item)

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
   # Requires the following methods to be defined in the including class
   #   1. item (required)
   #   2. options (can be nil)
-  #   3. worker_class (required, can be anything)
+  #   3. job_class (required, can be anything)
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
@@ -46,10 +46,8 @@ module SidekiqUniqueJobs
     # @return [Class]
     #
     def lock_class
-      @lock_class ||= begin
-        locks.fetch(lock_type.to_sym) do
-          raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
-        end
+      @lock_class ||= locks.fetch(lock_type.to_sym) do
+        raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
       end
     end
 
@@ -57,7 +55,7 @@ module SidekiqUniqueJobs
     # The type of lock for this worker
     #
     #
-    # @return [Symbol]
+    # @return [Symbol, NilClass]
     #
     def lock_type
       @lock_type ||= options[LOCK] || item[LOCK]
@@ -71,8 +69,8 @@ module SidekiqUniqueJobs
     #
     def options
       @options ||= begin
-        opts = default_worker_options.dup
-        opts.merge!(worker_options) if sidekiq_worker_class?
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
         (opts || {}).stringify_keys
       end
     end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -20,7 +20,7 @@ module SidekiqUniqueJobs
         call_script(
           :reap_orphans,
           conn,
-          keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
           argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "concurrent/version"
+
 module SidekiqUniqueJobs
   module Orphans
     #
@@ -10,10 +12,18 @@ module SidekiqUniqueJobs
     module Manager
       module_function
 
+      #
+      # @return [Float] the amount to add to the reaper interval
       DRIFT_FACTOR = 0.02
+      #
+      # @return [Symbol] allowed reapers (:ruby or :lua)
       REAPERS      = [:ruby, :lua].freeze
 
+      # includes "SidekiqUniqueJobs::Connection"
+      # @!parse include SidekiqUniqueJobs::Connection
       include SidekiqUniqueJobs::Connection
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::Logging
 
       #
@@ -62,9 +72,15 @@ module SidekiqUniqueJobs
       # @return [<type>] <description>
       #
       def task
-        @task ||= default_task
+        @task ||= default_task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
       end
 
+      #
+      # A properly configured timer task
+      #
+      #
+      # @return [SidekiqUniqueJobs::TimerTask]
+      #
       def default_task
         SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
           with_logging_context do
@@ -76,8 +92,15 @@ module SidekiqUniqueJobs
         end
       end
 
+      #
+      # Store a task to use for scheduled execution
+      #
+      # @param [SidekiqUniqueJobs::TimerTask] task the task to use
+      #
+      # @return [void]
+      #
       def task=(task)
-        @task = task
+        @task = task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
       end
 
       #
@@ -87,9 +110,7 @@ module SidekiqUniqueJobs
       # @return [Hash]
       #
       def timer_task_options
-        { run_now: true,
-          execution_interval: reaper_interval,
-          timeout_interval: reaper_timeout }
+        { run_now: true, execution_interval: reaper_interval }
       end
 
       #
@@ -106,13 +127,6 @@ module SidekiqUniqueJobs
         SidekiqUniqueJobs.config.reaper_interval
       end
 
-      #
-      # @see SidekiqUniqueJobs::Config#reaper_timeout
-      #
-      def reaper_timeout
-        SidekiqUniqueJobs.config.reaper_timeout
-      end
-
       #
       # A context to use for all log entries
       #
@@ -201,10 +215,24 @@ module SidekiqUniqueJobs
         redis { |conn| conn.del(UNIQUE_REAPER) }
       end
 
+      #
+      # Reaper interval with a little drift
+      #   Redis isn't exact enough so to give a little bufffer,
+      #   we add a tiny value to the reaper interval.
+      #
+      #
+      # @return [Integer] <description>
+      #
       def drift_reaper_interval
         reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
       end
 
+      #
+      # Current time (as integer value)
+      #
+      #
+      # @return [Integer]
+      #
       def current_timestamp
         Time.now.to_i
       end

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -9,9 +9,17 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
+    # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
+      #
+      # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
       #
+      # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
+      MAX_QUEUE_LENGTH = 1000
+      #
       # @!attribute [r] digests
       #   @return [SidekiqUniqueJobs::Digests] digest collection
       attr_reader :digests
@@ -24,6 +32,22 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -31,9 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -43,22 +70,59 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of reaped locks
       #
       def call
+        return if queues_very_full?
+
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && VersionCheck.satisfied?(::Redis::VERSION, ">= 4.6.0")
+          conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+        else
+          conn.zrangebyscore(EXPIRING_DIGESTS, 0, max_score)
+        end
+      end
+
       #
       # Find orphaned digests
       #
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
-          next if belongs_to_job?(digest)
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        page = 0
+        per = reaper_count * 2
+        orphans = []
+        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+
+        while results.size.positive?
+          results.each do |digest|
+            break if timeout?
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
+
+          break if timeout?
+          break if orphans.size >= reaper_count
 
-          memo << digest
-          break if memo.size >= reaper_count
+          page += 1
+          results = conn.zrange(digests.key, page * per, (page + 1) * per)
         end
+
+        orphans
+      end
+
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_source
       end
 
       #
@@ -124,14 +188,14 @@ module SidekiqUniqueJobs
           return false if procs.empty?
 
           procs.sort.each do |key|
-            valid, workers = conn.pipelined do
+            valid, workers = conn.pipelined do |pipeline|
               # TODO: Remove the if statement in the future
-              if conn.respond_to?(:exists?)
-                conn.exists?(key)
+              if pipeline.respond_to?(:exists?)
+                pipeline.exists?(key)
               else
-                conn.exists(key)
+                pipeline.exists(key)
               end
-              conn.hgetall("#{key}:workers")
+              pipeline.hgetall("#{key}:work")
             end
 
             next unless valid
@@ -182,7 +246,8 @@ module SidekiqUniqueJobs
         page_size    = 50
 
         loop do
-          range_start = page * page_size - deleted_size
+          range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -192,7 +257,26 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
+        end
+      end
+
+      # If sidekiq queues are very full, it becomes highly inefficient for the reaper
+      # because it must check every queued job to verify a digest is safe to delete
+      # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
+      # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
+      def queues_very_full?
+        total_queue_size = 0
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            total_queue_size += conn.llen("queue:#{queue}")
+
+            return true if total_queue_size > MAX_QUEUE_LENGTH
+          end
         end
+        false
       end
 
       #
@@ -209,5 +293,6 @@ module SidekiqUniqueJobs
         conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/sidekiq_unique_jobs/redis/string.rb

@@ -25,7 +25,9 @@ module SidekiqUniqueJobs
       #
       # @return [true, false]
       #
-      def set(obj)
+      def set(obj, pipeline = nil)
+        return pipeline.set(key, obj) if pipeline
+
         redis { |conn| conn.set(key, obj) }
       end
 

data/lib/sidekiq_unique_jobs/reflectable.rb

@@ -7,11 +7,20 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   module Reflectable
-    def reflect(name, *args)
-      SidekiqUniqueJobs.reflections.dispatch(name, *args)
+    #
+    # Reflects on specific event
+    #
+    # @param [Symbol] reflection the reflected event
+    # @param [Array] args arguments to provide to reflector
+    #
+    # @return [void]
+    #
+    def reflect(reflection, *args)
+      SidekiqUniqueJobs.reflections.dispatch(reflection, *args)
       nil
     rescue UniqueJobsError => ex
       SidekiqUniqueJobs.logger.error(ex)
+      nil
     end
   end
 end

data/lib/sidekiq_unique_jobs/reflections.rb

@@ -42,6 +42,14 @@ module SidekiqUniqueJobs
       @reflections = {}
     end
 
+    #
+    # Dispatch a reflected event
+    #
+    # @param [reflection] reflection the reflected event
+    # @param [Array] args the arguments to provide to the block
+    #
+    # @return [void] <description>
+    #
     def dispatch(reflection, *args)
       if (block = @reflections[reflection])
         block.call(*args)
@@ -49,12 +57,15 @@ module SidekiqUniqueJobs
         if DEPRECATIONS.key?(reflection)
           replacement, removal_version = DEPRECATIONS[reflection]
           SidekiqUniqueJobs::Deprecation.warn(
-            "#{reflection} is deprecated and will be removed in version #{removal_version}. Use #{replacement} instead.",
+            "#{reflection} is deprecated and will be removed in version #{removal_version}." \
+            " Use #{replacement} instead.",
           )
         end
       elsif misconfigured?(reflection)
         raise NoSuchNotificationError, reflection
       end
+
+      nil
     end
 
     def configured?(reflection)

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -54,13 +54,13 @@ module SidekiqUniqueJobs
       # Only used to reduce a little bit of duplication
       # @see call_script
       def do_call(file_name, conn, keys, argv)
-        argv = argv.dup.concat([
-                                 now_f,
-                                 debug_lua,
-                                 max_history,
-                                 file_name,
-                                 redis_version,
-                               ])
+        argv = argv.dup.push(
+          now_f,
+          debug_lua,
+          max_history,
+          file_name,
+          redis_version,
+        )
         Script.execute(file_name, conn, keys: keys, argv: argv)
       end
 

data/lib/sidekiq_unique_jobs/server.rb

@@ -5,7 +5,7 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   class Server
-    DEATH_HANDLER ||= (lambda do |job, _ex|
+    DEATH_HANDLER = (lambda do |job, _ex|
       return unless (digest = job["lock_digest"])
 
       SidekiqUniqueJobs::Digests.new.delete_by_digest(digest)
@@ -25,6 +25,12 @@ module SidekiqUniqueJobs
       config.death_handlers << death_handler
     end
 
+    #
+    # Start the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
     def self.start
       SidekiqUniqueJobs::UpdateVersion.call
       SidekiqUniqueJobs::UpgradeLocks.call
@@ -32,6 +38,12 @@ module SidekiqUniqueJobs
       SidekiqUniqueJobs::Orphans::ReaperResurrector.start
     end
 
+    #
+    # Stop the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
     def self.stop
       SidekiqUniqueJobs::Orphans::Manager.stop
     end