sidekiq-unique-jobs 7.1.19 → 7.1.29

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -32,6 +32,7 @@ module SidekiqUniqueJobs
     #
     # @return [Float] used to take into consideration the inaccuracy of redis timestamps
     CLOCK_DRIFT_FACTOR = 0.01
+    NETWORK_FACTOR = 0.04
 
     #
     # @!attribute [r] key
@@ -184,22 +185,23 @@ module SidekiqUniqueJobs
     #
     # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
     # @param [Method] primed_method reference to the method to use for getting a primed token
+    # @param [nil, Integer, Float] time to wait before timeout
     #
     # @yieldparam [string] job_id the sidekiq JID
     # @yieldreturn [void] whatever the calling block returns
     def lock!(conn, primed_method, wait = nil)
-      return yield job_id if locked?(conn)
+      return yield if locked?(conn)
 
       enqueue(conn) do |queued_jid|
-        reflect(:debug, item, queued_jid)
+        reflect(:debug, :queued, item, queued_jid)
 
         primed_method.call(conn, wait) do |primed_jid|
           reflect(:debug, :primed, item, primed_jid)
-
           locked_jid = call_script(:lock, key.to_a, argv, conn)
+
           if locked_jid
             reflect(:debug, :locked, item, locked_jid)
-            return yield job_id
+            return yield
           end
         end
       end
@@ -239,9 +241,18 @@ module SidekiqUniqueJobs
     # @return [Object] whatever the block returns when lock was acquired
     #
     def primed_async(conn, wait = nil, &block)
+      timeout = (wait || config.timeout).to_i
+      timeout = 1 if timeout.zero?
+
+      brpoplpush_timeout = timeout
+      concurrent_timeout = add_drift(timeout)
+
+      reflect(:debug, :timeouts, item,
+              timeouts: { brpoplpush_timeout: brpoplpush_timeout, concurrent_timeout: concurrent_timeout })
+
       primed_jid = Concurrent::Promises
-                   .future(conn) { |red_con| pop_queued(red_con, wait) }
-                   .value(add_drift(wait || config.timeout))
+                   .future(conn) { |red_con| pop_queued(red_con, timeout) }
+                   .value
 
       handle_primed(primed_jid, &block)
     end
@@ -273,7 +284,7 @@ module SidekiqUniqueJobs
     #
     # @return [String] a previously enqueued token (now taken off the queue)
     #
-    def pop_queued(conn, wait = nil)
+    def pop_queued(conn, wait = 1)
       wait ||= config.timeout if config.wait_for_lock?
 
       if wait.nil?
@@ -287,10 +298,15 @@ module SidekiqUniqueJobs
     # @api private
     #
     def brpoplpush(conn, wait)
+      # passing timeout 0 to brpoplpush causes it to block indefinitely
       raise InvalidArgument, "wait must be an integer" unless wait.is_a?(Integer)
+      return conn.brpoplpush(key.queued, key.primed, wait) if conn.class.to_s == "Redis::Namespace"
 
-      # passing timeout 0 to brpoplpush causes it to block indefinitely
-      conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && conn.respond_to?(:blmove)
+        conn.blmove(key.queued, key.primed, "RIGHT", "LEFT", timeout: wait)
+      else
+        conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      end
     end
 
     #
@@ -359,5 +375,9 @@ module SidekiqUniqueJobs
         TIME => now_f,
       )
     end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
   end
 end

data/lib/sidekiq_unique_jobs/logging.rb

@@ -30,6 +30,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_debug(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.debug(message, &block)
       nil
@@ -45,6 +47,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_info(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.info(message, &block)
       nil
@@ -60,6 +64,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_warn(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.warn(message, &block)
       nil
@@ -75,6 +81,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_error(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.error(message, &block)
       nil
@@ -90,6 +98,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_fatal(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.fatal(message, &block)
 
@@ -218,5 +228,9 @@ module SidekiqUniqueJobs
 
       yield
     end
+
+    def logging?
+      SidekiqUniqueJobs.logging?
+    end
   end
 end

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -1,11 +1,12 @@
 -------- BEGIN keys ---------
-local digest    = KEYS[1]
-local queued    = KEYS[2]
-local primed    = KEYS[3]
-local locked    = KEYS[4]
-local info      = KEYS[5]
-local changelog = KEYS[6]
-local digests   = KEYS[7]
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
 -------- END keys ---------
 
 
@@ -57,8 +58,13 @@ if limit_exceeded then
   return nil
 end
 
-log_debug("ZADD", digests, current_time, digest)
-redis.call("ZADD", digests, current_time, digest)
+if lock_type == "until_expired" and pttl and pttl > 0 then
+  log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+  redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+else
+  log_debug("ZADD", digests, current_time, digest)
+  redis.call("ZADD", digests, current_time, digest)
+end
 
 log_debug("HSET", locked, job_id, current_time)
 redis.call("HSET", locked, job_id, current_time)

data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua

@@ -0,0 +1,92 @@
+-------- BEGIN keys ---------
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  BEGIN lock.lua ---------
+log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug(locked, "already locked with job_id:", job_id)
+  log("Duplicate")
+
+  log_debug("LREM", queued, -1, job_id)
+  redis.call("LREM", queued, -1, job_id)
+
+  log_debug("LREM", primed, 1, job_id)
+  redis.call("LREM", primed, 1, job_id)
+
+  return job_id
+end
+
+local locked_count   = redis.call("HLEN", locked)
+local within_limit   = limit > locked_count
+local limit_exceeded = not within_limit
+
+if limit_exceeded then
+  log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+  log("Limited")
+  return nil
+end
+
+log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+
+log_debug("HSET", locked, job_id, current_time)
+redis.call("HSET", locked, job_id, current_time)
+
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, 1, job_id)
+redis.call("LREM", primed, 1, job_id)
+
+-- The Sidekiq client sets pttl
+log_debug("PEXPIRE", digest, pttl)
+redis.call("PEXPIRE", digest, pttl)
+
+log_debug("PEXPIRE", locked, pttl)
+redis.call("PEXPIRE", locked, pttl)
+
+log_debug("PEXPIRE", info, pttl)
+redis.call("PEXPIRE", info, pttl)
+
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log("Locked")
+log_debug("END lock digest:", digest, "job_id:", job_id)
+return job_id
+----------  END lock.lua  ----------

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -1,9 +1,10 @@
 redis.replicate_commands()
 
 -------- BEGIN keys ---------
-local digests_set  = KEYS[1]
-local schedule_set = KEYS[2]
-local retry_set    = KEYS[3]
+local digests_set          = KEYS[1]
+local expiring_digests_set = KEYS[2]
+local schedule_set         = KEYS[3]
+local retry_set            = KEYS[4]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
@@ -90,5 +91,32 @@ repeat
   index = index + per
 until index >= total or del_count >= reaper_count
 
+if del_count < reaper_count then
+  index = 0
+  total = redis.call("ZCOUNT", expiring_digests_set, 0, current_time)
+  repeat
+    local digests = redis.call("ZRANGEBYSCORE", expiring_digests_set, 0, current_time, "LIMIT", index, index + per -1)
+
+    for _, digest in pairs(digests) do
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", expiring_digests_set, digest)
+      del_count = del_count + 1
+    end
+
+    index = index + per
+  until index >= total or del_count >= reaper_count
+end
+
 log_debug("END")
 return del_count

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -85,6 +85,11 @@ if locked_count and locked_count < 1 then
   redis.call(del_cmd, locked)
 end
 
+if redis.call("LLEN", primed) == 0 then
+  log_debug(del_cmd, primed)
+  redis.call(del_cmd, primed)
+end
+
 if limit and limit <= 1 and locked_count and locked_count <= 1 then
   log_debug("ZREM", digests, digest)
   redis.call("ZREM", digests, digest)

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
+      include Sidekiq::ClientMiddleware if defined?(Sidekiq::ClientMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -29,10 +29,10 @@ module SidekiqUniqueJobs
     # @yieldparam [<type>] if <description>
     # @yieldreturn [<type>] <describe what yield should return>
     def call(worker_class, item, queue, redis_pool = nil)
-      @worker_class = worker_class
-      @item         = item
-      @queue        = queue
-      @redis_pool   = redis_pool
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = worker_class
       return yield if unique_disabled?
 
       SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -14,14 +14,14 @@ module SidekiqUniqueJobs
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @worker_class = item[CLASS]
+        self.job_class = item[CLASS]
       end
 
       # Create a new job from the current one.
       #   This will mess up sidekiq stats because a new job is created
       def call
-        if sidekiq_worker_class?
-          if worker_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
             reflect(:rescheduled, item)
           else
             reflect(:reschedule_failed, item)

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
   # Requires the following methods to be defined in the including class
   #   1. item (required)
   #   2. options (can be nil)
-  #   3. worker_class (required, can be anything)
+  #   3. job_class (required, can be anything)
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
@@ -69,8 +69,8 @@ module SidekiqUniqueJobs
     #
     def options
       @options ||= begin
-        opts = default_worker_options.dup
-        opts.merge!(worker_options) if sidekiq_worker_class?
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
         (opts || {}).stringify_keys
       end
     end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -20,7 +20,7 @@ module SidekiqUniqueJobs
         call_script(
           :reap_orphans,
           conn,
-          keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
           argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -110,13 +110,7 @@ module SidekiqUniqueJobs
       # @return [Hash]
       #
       def timer_task_options
-        timer_task_options = { run_now: true, execution_interval: reaper_interval }
-
-        if VersionCheck.satisfied?(::Concurrent::VERSION, "< 1.1.10")
-          timer_task_options[:timeout_interval] = reaper_timeout
-        end
-
-        timer_task_options
+        { run_now: true, execution_interval: reaper_interval }
       end
 
       #
@@ -133,13 +127,6 @@ module SidekiqUniqueJobs
         SidekiqUniqueJobs.config.reaper_interval
       end
 
-      #
-      # @see SidekiqUniqueJobs::Config#reaper_timeout
-      #
-      def reaper_timeout
-        SidekiqUniqueJobs.config.reaper_timeout
-      end
-
       #
       # A context to use for all log entries
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -11,6 +11,8 @@ module SidekiqUniqueJobs
     #
     # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
@@ -30,6 +32,22 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -37,9 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -51,16 +72,27 @@ module SidekiqUniqueJobs
       def call
         return if queues_very_full?
 
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && VersionCheck.satisfied?(::Redis::VERSION, ">= 4.6.0")
+          conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+        else
+          conn.zrangebyscore(EXPIRING_DIGESTS, 0, max_score)
+        end
+      end
+
       #
       # Find orphaned digests
       #
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans # rubocop:disable Metrics/MethodLength
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
         page = 0
         per = reaper_count * 2
         orphans = []
@@ -68,12 +100,14 @@ module SidekiqUniqueJobs
 
         while results.size.positive?
           results.each do |digest|
+            break if timeout?
             next if belongs_to_job?(digest)
 
             orphans << digest
             break if orphans.size >= reaper_count
           end
 
+          break if timeout?
           break if orphans.size >= reaper_count
 
           page += 1
@@ -83,6 +117,14 @@ module SidekiqUniqueJobs
         orphans
       end
 
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_source
+      end
+
       #
       # Checks if the digest has a matching job.
       #   1. It checks the scheduled set
@@ -205,6 +247,7 @@ module SidekiqUniqueJobs
 
         loop do
           range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -214,6 +257,9 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
         end
       end
 

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -71,6 +71,16 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
     config.logger = other
   end
 
+  #
+  # Check if logging is enabled
+  #
+  #
+  # @return [true, false]
+  #
+  def logging?
+    config.logger_enabled
+  end
+
   #
   # Temporarily use another configuration and reset to the old config after yielding
   #