sidekiq-unique-jobs 7.1.20 → 7.1.30

data/lib/sidekiq_unique_jobs/lock_ttl.rb

@@ -33,8 +33,8 @@ module SidekiqUniqueJobs
     # @option item [String] :class the class of the sidekiq worker
     # @option item [Float] :at the unix time the job is scheduled at
     def initialize(item)
-      @item         = item
-      @worker_class = item[CLASS]
+      @item = item
+      self.job_class = item[CLASS]
     end
 
     #
@@ -67,9 +67,9 @@ module SidekiqUniqueJobs
     #
     def calculate
       ttl = item[LOCK_TTL]
-      ttl ||= worker_options[LOCK_TTL]
+      ttl ||= job_options[LOCK_TTL]
       ttl ||= item[LOCK_EXPIRATION] # TODO: Deprecate at some point
-      ttl ||= worker_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
+      ttl ||= job_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
       ttl ||= SidekiqUniqueJobs.config.lock_ttl
       ttl && (ttl.to_i + time_until_scheduled)
     end

data/lib/sidekiq_unique_jobs/lock_type.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Calculates the lock type
+  #
+  class LockType
+    # includes "SidekiqUniqueJobs::SidekiqWorkerMethods"
+    # @!parse include SidekiqUniqueJobs::SidekiqWorkerMethods
+    include SidekiqUniqueJobs::SidekiqWorkerMethods
+
+    #
+    # Computes lock type from job arguments, sidekiq_options.
+    #
+    # @return [Symbol] the lock type
+    # @return [NilClass] if no lock type is found.
+    #
+    def self.call(item)
+      new(item).call
+    end
+
+    # @!attribute [r] item
+    #   @return [Hash] the Sidekiq job hash
+    attr_reader :item
+
+    # @param [Hash] item the Sidekiq job hash
+    # @option item [Symbol, nil] :lock the type of lock to use.
+    # @option item [String] :class the class of the sidekiq worker
+    def initialize(item)
+      @item = item
+      self.job_class = item[CLASS]
+    end
+
+    def call
+      item[LOCK] || job_options[LOCK] || default_job_options[LOCK]
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -32,6 +32,7 @@ module SidekiqUniqueJobs
     #
     # @return [Float] used to take into consideration the inaccuracy of redis timestamps
     CLOCK_DRIFT_FACTOR = 0.01
+    NETWORK_FACTOR = 0.04
 
     #
     # @!attribute [r] key
@@ -126,7 +127,10 @@ module SidekiqUniqueJobs
     #
     def unlock!(conn = nil)
       call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
-        reflect(:debug, :unlocked, item, unlocked_jid) if unlocked_jid == job_id
+        if unlocked_jid == job_id
+          reflect(:debug, :unlocked, item, unlocked_jid)
+          reflect(:unlocked, item)
+        end
 
         unlocked_jid
       end
@@ -184,22 +188,23 @@ module SidekiqUniqueJobs
     #
     # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
     # @param [Method] primed_method reference to the method to use for getting a primed token
+    # @param [nil, Integer, Float] time to wait before timeout
     #
     # @yieldparam [string] job_id the sidekiq JID
     # @yieldreturn [void] whatever the calling block returns
     def lock!(conn, primed_method, wait = nil)
-      return yield job_id if locked?(conn)
+      return yield if locked?(conn)
 
       enqueue(conn) do |queued_jid|
-        reflect(:debug, item, queued_jid)
+        reflect(:debug, :queued, item, queued_jid)
 
         primed_method.call(conn, wait) do |primed_jid|
           reflect(:debug, :primed, item, primed_jid)
-
           locked_jid = call_script(:lock, key.to_a, argv, conn)
+
           if locked_jid
             reflect(:debug, :locked, item, locked_jid)
-            return yield job_id
+            return yield
           end
         end
       end
@@ -239,9 +244,22 @@ module SidekiqUniqueJobs
     # @return [Object] whatever the block returns when lock was acquired
     #
     def primed_async(conn, wait = nil, &block)
+      timeout = (wait || config.timeout).to_i
+      timeout = 1 if timeout.zero?
+
+      brpoplpush_timeout = timeout
+      concurrent_timeout = add_drift(timeout)
+
+      reflect(:debug, :timeouts, item,
+              timeouts: {
+                brpoplpush_timeout: brpoplpush_timeout,
+                concurrent_timeout: concurrent_timeout,
+              })
+
+      # NOTE: When debugging, change .value to .value!
       primed_jid = Concurrent::Promises
-                   .future(conn) { |red_con| pop_queued(red_con, wait) }
-                   .value(add_drift(wait || config.timeout))
+                   .future(conn) { |red_con| pop_queued(red_con, timeout) }
+                   .value
 
       handle_primed(primed_jid, &block)
     end
@@ -273,7 +291,7 @@ module SidekiqUniqueJobs
     #
     # @return [String] a previously enqueued token (now taken off the queue)
     #
-    def pop_queued(conn, wait = nil)
+    def pop_queued(conn, wait = 1)
       wait ||= config.timeout if config.wait_for_lock?
 
       if wait.nil?
@@ -287,10 +305,18 @@ module SidekiqUniqueJobs
     # @api private
     #
     def brpoplpush(conn, wait)
+      # passing timeout 0 to brpoplpush causes it to block indefinitely
       raise InvalidArgument, "wait must be an integer" unless wait.is_a?(Integer)
 
-      # passing timeout 0 to brpoplpush causes it to block indefinitely
-      conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      if defined?(::Redis::Namespace) && conn.instance_of?(::Redis::Namespace)
+        return conn.brpoplpush(key.queued, key.primed, wait)
+      end
+
+      if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && conn.respond_to?(:blmove)
+        conn.blmove(key.queued, key.primed, "RIGHT", "LEFT", timeout: wait)
+      else
+        conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      end
     end
 
     #
@@ -359,5 +385,9 @@ module SidekiqUniqueJobs
         TIME => now_f,
       )
     end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
   end
 end

data/lib/sidekiq_unique_jobs/logging.rb

@@ -30,6 +30,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_debug(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.debug(message, &block)
       nil
@@ -45,6 +47,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_info(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.info(message, &block)
       nil
@@ -60,6 +64,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_warn(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.warn(message, &block)
       nil
@@ -75,6 +81,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_error(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.error(message, &block)
       nil
@@ -90,6 +98,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_fatal(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.fatal(message, &block)
 
@@ -218,5 +228,9 @@ module SidekiqUniqueJobs
 
       yield
     end
+
+    def logging?
+      SidekiqUniqueJobs.logging?
+    end
   end
 end

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -1,11 +1,12 @@
 -------- BEGIN keys ---------
-local digest    = KEYS[1]
-local queued    = KEYS[2]
-local primed    = KEYS[3]
-local locked    = KEYS[4]
-local info      = KEYS[5]
-local changelog = KEYS[6]
-local digests   = KEYS[7]
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
 -------- END keys ---------
 
 
@@ -57,8 +58,13 @@ if limit_exceeded then
   return nil
 end
 
-log_debug("ZADD", digests, current_time, digest)
-redis.call("ZADD", digests, current_time, digest)
+if lock_type == "until_expired" and pttl and pttl > 0 then
+  log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+  redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+else
+  log_debug("ZADD", digests, current_time, digest)
+  redis.call("ZADD", digests, current_time, digest)
+end
 
 log_debug("HSET", locked, job_id, current_time)
 redis.call("HSET", locked, job_id, current_time)

data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua

@@ -0,0 +1,92 @@
+-------- BEGIN keys ---------
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  BEGIN lock.lua ---------
+log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug(locked, "already locked with job_id:", job_id)
+  log("Duplicate")
+
+  log_debug("LREM", queued, -1, job_id)
+  redis.call("LREM", queued, -1, job_id)
+
+  log_debug("LREM", primed, 1, job_id)
+  redis.call("LREM", primed, 1, job_id)
+
+  return job_id
+end
+
+local locked_count   = redis.call("HLEN", locked)
+local within_limit   = limit > locked_count
+local limit_exceeded = not within_limit
+
+if limit_exceeded then
+  log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+  log("Limited")
+  return nil
+end
+
+log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+
+log_debug("HSET", locked, job_id, current_time)
+redis.call("HSET", locked, job_id, current_time)
+
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, 1, job_id)
+redis.call("LREM", primed, 1, job_id)
+
+-- The Sidekiq client sets pttl
+log_debug("PEXPIRE", digest, pttl)
+redis.call("PEXPIRE", digest, pttl)
+
+log_debug("PEXPIRE", locked, pttl)
+redis.call("PEXPIRE", locked, pttl)
+
+log_debug("PEXPIRE", info, pttl)
+redis.call("PEXPIRE", info, pttl)
+
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log("Locked")
+log_debug("END lock digest:", digest, "job_id:", job_id)
+return job_id
+----------  END lock.lua  ----------

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -1,9 +1,10 @@
 redis.replicate_commands()
 
 -------- BEGIN keys ---------
-local digests_set  = KEYS[1]
-local schedule_set = KEYS[2]
-local retry_set    = KEYS[3]
+local digests_set          = KEYS[1]
+local expiring_digests_set = KEYS[2]
+local schedule_set         = KEYS[3]
+local retry_set            = KEYS[4]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
@@ -90,5 +91,32 @@ repeat
   index = index + per
 until index >= total or del_count >= reaper_count
 
+if del_count < reaper_count then
+  index = 0
+  total = redis.call("ZCOUNT", expiring_digests_set, 0, current_time)
+  repeat
+    local digests = redis.call("ZRANGEBYSCORE", expiring_digests_set, 0, current_time, "LIMIT", index, index + per -1)
+
+    for _, digest in pairs(digests) do
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", expiring_digests_set, digest)
+      del_count = del_count + 1
+    end
+
+    index = index + per
+  until index >= total or del_count >= reaper_count
+end
+
 log_debug("END")
 return del_count

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -15,7 +15,7 @@ local function find_digest_in_process_set(digest, threshold)
     log_debug("Found number of processes:", #processes, "next cursor:", next_process_cursor)
 
     for _, process in ipairs(processes) do
-      local workers_key = process .. ":workers"
+      local workers_key = process .. ":work"
       log_debug("searching in process set:", process,
                 "for digest:", digest,
                 "cursor:", process_cursor)

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -85,6 +85,11 @@ if locked_count and locked_count < 1 then
   redis.call(del_cmd, locked)
 end
 
+if redis.call("LLEN", primed) == 0 then
+  log_debug(del_cmd, primed)
+  redis.call(del_cmd, primed)
+end
+
 if limit and limit <= 1 and locked_count and locked_count <= 1 then
   log_debug("ZREM", digests, digest)
   redis.call("ZREM", digests, digest)

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
+      include Sidekiq::ClientMiddleware if defined?(Sidekiq::ClientMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -29,10 +29,10 @@ module SidekiqUniqueJobs
     # @yieldparam [<type>] if <description>
     # @yieldreturn [<type>] <describe what yield should return>
     def call(worker_class, item, queue, redis_pool = nil)
-      @worker_class = worker_class
-      @item         = item
-      @queue        = queue
-      @redis_pool   = redis_pool
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = worker_class
       return yield if unique_disabled?
 
       SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -14,14 +14,14 @@ module SidekiqUniqueJobs
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @worker_class = item[CLASS]
+        self.job_class = item[CLASS]
       end
 
       # Create a new job from the current one.
       #   This will mess up sidekiq stats because a new job is created
       def call
-        if sidekiq_worker_class?
-          if worker_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
             reflect(:rescheduled, item)
           else
             reflect(:reschedule_failed, item)

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
   # Requires the following methods to be defined in the including class
   #   1. item (required)
   #   2. options (can be nil)
-  #   3. worker_class (required, can be anything)
+  #   3. job_class (required, can be anything)
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
@@ -55,7 +55,7 @@ module SidekiqUniqueJobs
     # The type of lock for this worker
     #
     #
-    # @return [Symbol]
+    # @return [Symbol, NilClass]
     #
     def lock_type
       @lock_type ||= options[LOCK] || item[LOCK]
@@ -69,8 +69,8 @@ module SidekiqUniqueJobs
     #
     def options
       @options ||= begin
-        opts = default_worker_options.dup
-        opts.merge!(worker_options) if sidekiq_worker_class?
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
         (opts || {}).stringify_keys
       end
     end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -20,7 +20,7 @@ module SidekiqUniqueJobs
         call_script(
           :reap_orphans,
           conn,
-          keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
           argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -72,7 +72,7 @@ module SidekiqUniqueJobs
       # @return [<type>] <description>
       #
       def task
-        @task ||= default_task
+        @task ||= default_task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
       end
 
       #
@@ -100,7 +100,7 @@ module SidekiqUniqueJobs
       # @return [void]
       #
       def task=(task)
-        @task = task
+        @task = task # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
       end
 
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -34,9 +34,15 @@ module SidekiqUniqueJobs
 
       #
       # @!attribute [r] start_time
-      #   @return [Integer] The timestamp this execution started represented as integer
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
       attr_reader :start_time
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
       #
       # @!attribute [r] timeout_ms
       #   @return [Integer] The allowed ms before timeout
@@ -49,11 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests    = SidekiqUniqueJobs::Digests.new
-        @scheduled  = Redis::SortedSet.new(SCHEDULE)
-        @retried    = Redis::SortedSet.new(RETRY)
-        @start_time = time_source.call
-        @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -65,9 +72,20 @@ module SidekiqUniqueJobs
       def call
         return if queues_very_full?
 
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && VersionCheck.satisfied?(::Redis::VERSION, ">= 4.6.0")
+          conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+        else
+          conn.zrangebyscore(EXPIRING_DIGESTS, 0, max_score)
+        end
+      end
+
       #
       # Find orphaned digests
       #
@@ -104,7 +122,7 @@ module SidekiqUniqueJobs
       end
 
       def elapsed_ms
-        time_source.call - start_time
+        time_source.call - start_source
       end
 
       #
@@ -177,7 +195,7 @@ module SidekiqUniqueJobs
               else
                 pipeline.exists(key)
               end
-              pipeline.hgetall("#{key}:workers")
+              pipeline.hgetall("#{key}:work")
             end
 
             next unless valid
@@ -229,6 +247,7 @@ module SidekiqUniqueJobs
 
         loop do
           range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -238,6 +257,9 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
         end
       end
 

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -54,13 +54,13 @@ module SidekiqUniqueJobs
       # Only used to reduce a little bit of duplication
       # @see call_script
       def do_call(file_name, conn, keys, argv)
-        argv = argv.dup.concat([
-                                 now_f,
-                                 debug_lua,
-                                 max_history,
-                                 file_name,
-                                 redis_version,
-                               ])
+        argv = argv.dup.push(
+          now_f,
+          debug_lua,
+          max_history,
+          file_name,
+          redis_version,
+        )
         Script.execute(file_name, conn, keys: keys, argv: argv)
       end