sidekiq-unique-jobs 7.1.8 → 7.1.29

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -1,9 +1,10 @@
 redis.replicate_commands()
 
 -------- BEGIN keys ---------
-local digests_set  = KEYS[1]
-local schedule_set = KEYS[2]
-local retry_set    = KEYS[3]
+local digests_set          = KEYS[1]
+local expiring_digests_set = KEYS[2]
+local schedule_set         = KEYS[3]
+local retry_set            = KEYS[4]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
@@ -90,5 +91,32 @@ repeat
   index = index + per
 until index >= total or del_count >= reaper_count
 
+if del_count < reaper_count then
+  index = 0
+  total = redis.call("ZCOUNT", expiring_digests_set, 0, current_time)
+  repeat
+    local digests = redis.call("ZRANGEBYSCORE", expiring_digests_set, 0, current_time, "LIMIT", index, index + per -1)
+
+    for _, digest in pairs(digests) do
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", expiring_digests_set, digest)
+      del_count = del_count + 1
+    end
+
+    index = index + per
+  until index >= total or del_count >= reaper_count
+end
+
 log_debug("END")
 return del_count

data/lib/sidekiq_unique_jobs/lua/shared/_common.lua

@@ -31,7 +31,7 @@ local function log(message, prev_jid)
   log_debug("ZADD", changelog, current_time, entry);
   redis.call("ZADD", changelog, current_time, entry);
   local total_entries = redis.call("ZCARD", changelog)
-  local removed_entries = redis.call("ZREMRANGEBYRANK", changelog, max_history, -1)
+  local removed_entries = redis.call("ZREMRANGEBYRANK", changelog, 0, -1 * max_history)
   if removed_entries > 0 then
     log_debug("Removing", removed_entries , "entries from changelog (total entries", total_entries, "exceeds max_history:", max_history ..")");
   end

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -65,18 +65,12 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, -1, job_id)
 redis.call("LREM", primed, -1, job_id)
 
-if limit and limit <= 1 and locked_count and locked_count <= 1 then
-  log_debug("ZREM", digests, digest)
-  redis.call("ZREM", digests, digest)
-end
-
 local redis_version = toversion(redisversion)
 local del_cmd       = "DEL"
 
 if tonumber(redis_version["major"]) >= 4 then del_cmd =  "UNLINK"; end
 
 if lock_type ~= "until_expired" then
-
   log_debug(del_cmd, digest, info)
   redis.call(del_cmd, digest, info)
 
@@ -86,11 +80,21 @@ end
 
 local locked_count = redis.call("HLEN", locked)
 
-if tonumber(locked_count) < 1 then
+if locked_count and locked_count < 1 then
   log_debug(del_cmd, locked)
   redis.call(del_cmd, locked)
 end
 
+if redis.call("LLEN", primed) == 0 then
+  log_debug(del_cmd, primed)
+  redis.call(del_cmd, primed)
+end
+
+if limit and limit <= 1 and locked_count and locked_count <= 1 then
+  log_debug("ZREM", digests, digest)
+  redis.call("ZREM", digests, digest)
+end
+
 log_debug("LPUSH", queued, "1")
 redis.call("LPUSH", queued, "1")
 

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
+      include Sidekiq::ClientMiddleware if defined?(Sidekiq::ClientMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -29,10 +29,10 @@ module SidekiqUniqueJobs
     # @yieldparam [<type>] if <description>
     # @yieldreturn [<type>] <describe what yield should return>
     def call(worker_class, item, queue, redis_pool = nil)
-      @worker_class = worker_class
-      @item         = item
-      @queue        = queue
-      @redis_pool   = redis_pool
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = worker_class
       return yield if unique_disabled?
 
       SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -96,10 +96,10 @@ module SidekiqUniqueJobs
       #
       def push_to_deadset
         redis do |conn|
-          conn.multi do
-            conn.zadd("dead", now_f, payload)
-            conn.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
-            conn.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
+          conn.multi do |pipeline|
+            pipeline.zadd("dead", now_f, payload)
+            pipeline.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
+            pipeline.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
           end
         end
       end

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -14,14 +14,14 @@ module SidekiqUniqueJobs
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @worker_class = item[CLASS]
+        self.job_class = item[CLASS]
       end
 
       # Create a new job from the current one.
       #   This will mess up sidekiq stats because a new job is created
       def call
-        if sidekiq_worker_class?
-          if worker_class.perform_in(5, *item[ARGS])
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
             reflect(:rescheduled, item)
           else
             reflect(:reschedule_failed, item)

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
   # Requires the following methods to be defined in the including class
   #   1. item (required)
   #   2. options (can be nil)
-  #   3. worker_class (required, can be anything)
+  #   3. job_class (required, can be anything)
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
@@ -69,8 +69,8 @@ module SidekiqUniqueJobs
     #
     def options
       @options ||= begin
-        opts = default_worker_options.dup
-        opts.merge!(worker_options) if sidekiq_worker_class?
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
         (opts || {}).stringify_keys
       end
     end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -20,7 +20,7 @@ module SidekiqUniqueJobs
         call_script(
           :reap_orphans,
           conn,
-          keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
           argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "concurrent/version"
+
 module SidekiqUniqueJobs
   module Orphans
     #
@@ -108,9 +110,7 @@ module SidekiqUniqueJobs
       # @return [Hash]
       #
       def timer_task_options
-        { run_now: true,
-          execution_interval: reaper_interval,
-          timeout_interval: reaper_timeout }
+        { run_now: true, execution_interval: reaper_interval }
       end
 
       #
@@ -127,13 +127,6 @@ module SidekiqUniqueJobs
         SidekiqUniqueJobs.config.reaper_interval
       end
 
-      #
-      # @see SidekiqUniqueJobs::Config#reaper_timeout
-      #
-      def reaper_timeout
-        SidekiqUniqueJobs.config.reaper_timeout
-      end
-
       #
       # A context to use for all log entries
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -9,11 +9,17 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
+    # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
       #
+      # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
+      MAX_QUEUE_LENGTH = 1000
+      #
       # @!attribute [r] digests
       #   @return [SidekiqUniqueJobs::Digests] digest collection
       attr_reader :digests
@@ -26,6 +32,22 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -33,9 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -45,22 +70,59 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of reaped locks
       #
       def call
+        return if queues_very_full?
+
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && VersionCheck.satisfied?(::Redis::VERSION, ">= 4.6.0")
+          conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+        else
+          conn.zrangebyscore(EXPIRING_DIGESTS, 0, max_score)
+        end
+      end
+
       #
       # Find orphaned digests
       #
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
-          next if belongs_to_job?(digest)
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        page = 0
+        per = reaper_count * 2
+        orphans = []
+        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+
+        while results.size.positive?
+          results.each do |digest|
+            break if timeout?
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
 
-          memo << digest
-          break if memo.size >= reaper_count
+          break if timeout?
+          break if orphans.size >= reaper_count
+
+          page += 1
+          results = conn.zrange(digests.key, page * per, (page + 1) * per)
         end
+
+        orphans
+      end
+
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_source
       end
 
       #
@@ -126,14 +188,14 @@ module SidekiqUniqueJobs
           return false if procs.empty?
 
           procs.sort.each do |key|
-            valid, workers = conn.pipelined do
+            valid, workers = conn.pipelined do |pipeline|
               # TODO: Remove the if statement in the future
-              if conn.respond_to?(:exists?)
-                conn.exists?(key)
+              if pipeline.respond_to?(:exists?)
+                pipeline.exists?(key)
               else
-                conn.exists(key)
+                pipeline.exists(key)
               end
-              conn.hgetall("#{key}:workers")
+              pipeline.hgetall("#{key}:workers")
             end
 
             next unless valid
@@ -185,6 +247,7 @@ module SidekiqUniqueJobs
 
         loop do
           range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -194,7 +257,26 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
+        end
+      end
+
+      # If sidekiq queues are very full, it becomes highly inefficient for the reaper
+      # because it must check every queued job to verify a digest is safe to delete
+      # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
+      # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
+      def queues_very_full?
+        total_queue_size = 0
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            total_queue_size += conn.llen("queue:#{queue}")
+
+            return true if total_queue_size > MAX_QUEUE_LENGTH
+          end
         end
+        false
       end
 
       #
@@ -211,5 +293,6 @@ module SidekiqUniqueJobs
         conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/sidekiq_unique_jobs/redis/string.rb

@@ -25,7 +25,9 @@ module SidekiqUniqueJobs
       #
       # @return [true, false]
       #
-      def set(obj)
+      def set(obj, pipeline = nil)
+        return pipeline.set(key, obj) if pipeline
+
         redis { |conn| conn.set(key, obj) }
       end
 

data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb

@@ -68,7 +68,7 @@ module Sidekiq
     prepend UniqueExtension
   end
 
-  if Sidekiq.const_defined?("JobRecord")
+  if Sidekiq.const_defined?(:JobRecord)
     # See Sidekiq::Api
     class JobRecord
       #

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -71,6 +71,16 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
     config.logger = other
   end
 
+  #
+  # Check if logging is enabled
+  #
+  #
+  # @return [true, false]
+  #
+  def logging?
+    config.logger_enabled
+  end
+
   #
   # Temporarily use another configuration and reset to the old config after yielding
   #

data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb

@@ -5,41 +5,56 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module SidekiqWorkerMethods
+    #
+    # @!attribute [r] job_class
+    #   @return [Sidekiq::Worker] The Sidekiq::Worker implementation
+    attr_reader :job_class
+
     # Avoids duplicating worker_class.respond_to? in multiple places
     # @return [true, false]
-    def worker_method_defined?(method_sym)
-      worker_class.respond_to?(method_sym)
+    def job_method_defined?(method_sym)
+      job_class.respond_to?(method_sym)
     end
 
     # Wraps #get_sidekiq_options to always work with a hash
     # @return [Hash] of the worker class sidekiq options
-    def worker_options
-      return {} unless sidekiq_worker_class?
+    def job_options
+      return {} unless sidekiq_job_class?
 
-      worker_class.get_sidekiq_options.deep_stringify_keys
+      job_class.get_sidekiq_options.deep_stringify_keys
     end
 
     # Tests that the
-    # @return [true] if worker_class responds to get_sidekiq_options
-    # @return [false] if worker_class does not respond to get_sidekiq_options
-    def sidekiq_worker_class?
-      worker_method_defined?(:get_sidekiq_options)
+    # @return [true] if job_class responds to get_sidekiq_options
+    # @return [false] if job_class does not respond to get_sidekiq_options
+    def sidekiq_job_class?
+      job_method_defined?(:get_sidekiq_options)
     end
 
-    # The Sidekiq::Worker implementation
-    # @return [Sidekiq::Worker]
-    def worker_class
-      @_worker_class ||= worker_class_constantize # rubocop:disable Naming/MemoizedInstanceVariableName
+    def job_class=(obj)
+      # this is what was originally passed in, it can be an instance or a class depending on sidekiq version
+      @original_job_class = obj
+      @job_class = job_class_constantize(obj)
     end
 
     # The hook to call after a successful unlock
     # @return [Proc]
-    def after_unlock_hook
+    def after_unlock_hook # rubocop:disable Metrics/MethodLength
       lambda do
-        if @worker_class.respond_to?(:after_unlock)
-          @worker_class.after_unlock # instance method in sidekiq v6
-        elsif worker_class.respond_to?(:after_unlock)
-          worker_class.after_unlock # class method regardless of sidekiq version
+        if @original_job_class.respond_to?(:after_unlock)
+          # instance method in sidekiq v6
+          if @original_job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
+            @original_job_class.after_unlock(item)
+          else
+            @original_job_class.after_unlock
+          end
+        elsif job_class.respond_to?(:after_unlock)
+          # class method regardless of sidekiq version
+          if job_class.method(:after_unlock).arity.positive? # arity check to maintain backwards compatibility
+            job_class.after_unlock(item)
+          else
+            job_class.after_unlock
+          end
         end
       end
     end
@@ -48,7 +63,7 @@ module SidekiqUniqueJobs
     # failing back to the original argument when the constant can't be found
     #
     # @return [Sidekiq::Worker]
-    def worker_class_constantize(klazz = @worker_class)
+    def job_class_constantize(klazz = @job_class)
       SidekiqUniqueJobs.safe_constantize(klazz)
     end
 
@@ -58,8 +73,12 @@ module SidekiqUniqueJobs
     #
     # @return [Hash<Symbol, Object>]
     #
-    def default_worker_options
-      Sidekiq.default_worker_options
+    def default_job_options
+      if Sidekiq.respond_to?(:default_job_options)
+        Sidekiq.default_job_options
+      else
+        Sidekiq.default_worker_options
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/testing.rb

@@ -21,16 +21,24 @@ module Sidekiq
   #
   # @param [Hash<Symbol, Object>] tmp_config the temporary config to use
   #
-  def self.use_options(tmp_config = {})
-    old_options = default_worker_options.dup
+  def self.use_options(tmp_config = {}) # rubocop:disable Metrics/MethodLength
+    if respond_to?(:default_job_options)
+      default_job_options.clear
+      self.default_job_options = tmp_config
+    else
+      default_worker_options.clear
+      self.default_worker_options = tmp_config
+    end
 
-    default_worker_options.clear
-    self.default_worker_options = tmp_config
     yield
   ensure
-    default_worker_options.clear
-    self.default_worker_options = DEFAULT_WORKER_OPTIONS
-    self.default_worker_options = old_options
+    if respond_to?(:default_job_options)
+      default_job_options.clear
+      self.default_job_options = default_job_options
+    else
+      default_worker_options.clear
+      self.default_worker_options = DEFAULT_WORKER_OPTIONS
+    end
   end
 
   #
@@ -54,20 +62,14 @@ module Sidekiq
 
         yield
       ensure
-        self.sidekiq_options_hash = Sidekiq::DEFAULT_WORKER_OPTIONS
-        sidekiq_options(old_options)
-      end
+        self.sidekiq_options_hash =
+          if Sidekiq.respond_to?(:default_job_options)
+            Sidekiq.default_job_options
+          else
+            DEFAULT_WORKER_OPTIONS
+          end
 
-      #
-      # Clears the jobs for this worker and removes all locks
-      #
-      def clear
-        jobs.each do |job|
-          SidekiqUniqueJobs::Unlockable.unlock(job)
-        end
-
-        Sidekiq::Queues[queue].clear
-        jobs.clear
+        sidekiq_options(old_options)
       end
     end
 
@@ -88,6 +90,36 @@ module Sidekiq
         super(options)
       end
 
+      #
+      # Prepends deletion of locks to clear
+      #
+      module ClassMethods
+        #
+        # Clears the jobs for this worker and removes all locks
+        #
+        def clear
+          jobs.each do |job|
+            SidekiqUniqueJobs::Unlockable.unlock(job)
+          end
+
+          super
+        end
+      end
+    end
+
+    prepend Overrides
+
+    #
+    # Prepends methods to Sidekiq::Worker
+    #
+    module ClassMethods
+      prepend Overrides::ClassMethods
+    end
+
+    #
+    # Prepends singleton methods to Sidekiq::Worker
+    #
+    module SignletonOverrides
       #
       # Clears all jobs for this worker and removes all locks
       #
@@ -98,6 +130,6 @@ module Sidekiq
       end
     end
 
-    prepend Overrides
+    singleton_class.prepend SignletonOverrides
   end
 end