sidekiq-unique-jobs 7.0.2 → 7.1.12

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -11,7 +11,7 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class WhileExecuting < BaseLock
-      RUN_SUFFIX ||= ":RUN"
+      RUN_SUFFIX = ":RUN"
 
       include SidekiqUniqueJobs::OptionsWithFallback
       include SidekiqUniqueJobs::Logging::Middleware
@@ -29,21 +29,26 @@ module SidekiqUniqueJobs
       #   These locks should only ever be created in the server process.
       # @return [true] always returns true
       def lock
-        true
+        job_id = item[JID]
+        yield if block_given?
+
+        job_id
       end
 
       # Executes in the Sidekiq server process.
       #   These jobs are locked in the server process not from the client
       # @yield to the worker class perform method
-      def execute
+      def execute(&block)
         with_logging_context do
-          server_strategy&.call unless locksmith.lock do
+          executed = locksmith.execute do
             yield
-            callback_safely
+            callback_safely if locksmith.unlock
+          ensure
+            locksmith.unlock
           end
+
+          call_strategy(origin: :server, &block) unless executed
         end
-      ensure
-        locksmith.unlock!
       end
 
       private
@@ -51,7 +56,9 @@ module SidekiqUniqueJobs
       # This is safe as the base_lock always creates a new digest
       #   The append there for needs to be done every time
       def append_unique_key_suffix
-        item[LOCK_DIGEST] = item[LOCK_DIGEST] + RUN_SUFFIX
+        return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
+
+        item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -58,9 +58,9 @@ module SidekiqUniqueJobs
 
     def initialize(job_hash = {})
       @type        = job_hash[LOCK]&.to_sym
-      @worker      = job_hash[CLASS]
-      @limit       = job_hash.fetch(LOCK_LIMIT, 1)
-      @timeout     = job_hash.fetch(LOCK_TIMEOUT, 0)
+      @worker      = SidekiqUniqueJobs.safe_constantize(job_hash[CLASS])
+      @limit       = job_hash.fetch(LOCK_LIMIT, 1)&.to_i
+      @timeout     = job_hash.fetch(LOCK_TIMEOUT, 0)&.to_i
       @ttl         = job_hash.fetch(LOCK_TTL) { job_hash.fetch(LOCK_EXPIRATION, nil) }.to_i
       @pttl        = ttl * 1_000
       @lock_info   = job_hash.fetch(LOCK_INFO) { SidekiqUniqueJobs.config.lock_info }
@@ -71,11 +71,15 @@ module SidekiqUniqueJobs
       @on_server_conflict = job_hash[ON_SERVER_CONFLICT]
     end
 
+    def lock_info?
+      lock_info
+    end
+
     #
     # Indicate if timeout was set
     #
     #
-    # @return [true,fakse]
+    # @return [true,false]
     #
     def wait_for_lock?
       timeout.nil? || timeout.positive?

data/lib/sidekiq_unique_jobs/lock_ttl.rb

@@ -71,7 +71,7 @@ module SidekiqUniqueJobs
       ttl ||= item[LOCK_EXPIRATION] # TODO: Deprecate at some point
       ttl ||= worker_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
       ttl ||= SidekiqUniqueJobs.config.lock_ttl
-      ttl && ttl.to_i + time_until_scheduled
+      ttl && (ttl.to_i + time_until_scheduled)
     end
   end
 end

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -13,6 +13,10 @@ module SidekiqUniqueJobs
     # @!parse include SidekiqUniqueJobs::Logging
     include SidekiqUniqueJobs::Logging
 
+    # includes "SidekiqUniqueJobs::Reflectable"
+    # @!parse include SidekiqUniqueJobs::Reflectable
+    include SidekiqUniqueJobs::Reflectable
+
     # includes "SidekiqUniqueJobs::Timing"
     # @!parse include SidekiqUniqueJobs::Timing
     include SidekiqUniqueJobs::Timing
@@ -77,7 +81,7 @@ module SidekiqUniqueJobs
     # Deletes the lock regardless of if it has a pttl set
     #
     def delete!
-      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).positive?
+      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).to_i.positive?
     end
 
     #
@@ -85,16 +89,23 @@ module SidekiqUniqueJobs
     #
     # @return [String] the Sidekiq job_id that was locked/queued
     #
-    def lock(&block)
+    def lock(wait: nil)
+      method_name = wait ? :primed_async : :primed_sync
       redis(redis_pool) do |conn|
-        return lock_async(conn, &block) if block
-
-        lock_sync(conn) do
+        lock!(conn, method(method_name), wait) do
           return job_id
         end
       end
     end
 
+    def execute(&block)
+      raise SidekiqUniqueJobs::InvalidArgument, "#execute needs a block" unless block
+
+      redis(redis_pool) do |conn|
+        lock!(conn, method(:primed_async), &block)
+      end
+    end
+
     #
     # Removes the lock keys from Redis if locked by the provided jid/token
     #
@@ -114,11 +125,17 @@ module SidekiqUniqueJobs
     # @return [String] Sidekiq job_id (jid) if successful
     #
     def unlock!(conn = nil)
-      call_script(:unlock, key.to_a, argv, conn)
+      call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
+        reflect(:debug, :unlocked, item, unlocked_jid) if unlocked_jid == job_id
+
+        unlocked_jid
+      end
     end
 
     # Checks if this instance is considered locked
     #
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    #
     # @return [true, false] true when the :LOCKED hash contains the job_id
     #
     def locked?(conn = nil)
@@ -134,7 +151,7 @@ module SidekiqUniqueJobs
     # @return [String]
     #
     def to_s
-      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id}, locked=#{locked?})"
+      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id} locked=#{locked?})"
     end
 
     #
@@ -159,70 +176,74 @@ module SidekiqUniqueJobs
 
     attr_reader :redis_pool
 
-    def argv
-      [job_id, config.pttl, config.type, config.limit]
-    end
-
     #
-    # Used for runtime locks that need automatic unlock after yielding
+    # Used to reduce some duplication from the two methods
     #
-    # @param [Redis] conn a redis connection
-    #
-    # @return [nil] when lock was not possible
-    # @return [Object] whatever the block returns when lock was acquired
+    # @see lock
+    # @see execute
     #
-    # @yieldparam [String] job_id a Sidekiq JID
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    # @param [Method] primed_method reference to the method to use for getting a primed token
     #
-    def lock_async(conn)
+    # @yieldparam [string] job_id the sidekiq JID
+    # @yieldreturn [void] whatever the calling block returns
+    def lock!(conn, primed_method, wait = nil)
       return yield job_id if locked?(conn)
 
-      enqueue(conn) do
-        primed_async(conn) do
-          locked_token = call_script(:lock, key.to_a, argv, conn)
-          return yield if locked_token == job_id
+      enqueue(conn) do |queued_jid|
+        reflect(:debug, item, queued_jid)
+
+        primed_method.call(conn, wait) do |primed_jid|
+          reflect(:debug, :primed, item, primed_jid)
+
+          locked_jid = call_script(:lock, key.to_a, argv, conn)
+          if locked_jid
+            reflect(:debug, :locked, item, locked_jid)
+            return yield job_id
+          end
         end
       end
-    ensure
-      unlock!(conn)
     end
 
     #
-    # Pops an enqueued token
-    # @note Used for runtime locks to avoid problems with blocking commands
-    #   in current thread
+    # Prepares all the various lock data
     #
     # @param [Redis] conn a redis connection
     #
-    # @return [nil] when lock was not possible
-    # @return [Object] whatever the block returns when lock was acquired
+    # @return [nil] when redis was already prepared for this lock
+    # @return [yield<String>] when successfully enqueued
     #
-    def primed_async(conn)
-      return yield if Concurrent::Promises
-                      .future(conn) { |red_con| pop_queued(red_con) }
-                      .value(drift(config.ttl))
+    def enqueue(conn)
+      queued_jid, elapsed = timed do
+        call_script(:queue, key.to_a, argv, conn)
+      end
+
+      return unless queued_jid
+      return unless [job_id, "1"].include?(queued_jid)
+
+      validity = config.pttl - elapsed - drift(config.pttl)
+      return unless validity >= 0 || config.pttl.zero?
 
-      warn_about_timeout
+      write_lock_info(conn)
+      yield job_id
     end
 
     #
-    # Used for non-runtime locks (no block was given)
+    # Pops an enqueued token
+    # @note Used for runtime locks to avoid problems with blocking commands
+    #   in current thread
     #
     # @param [Redis] conn a redis connection
     #
     # @return [nil] when lock was not possible
     # @return [Object] whatever the block returns when lock was acquired
     #
-    # @yieldparam [String] job_id a Sidekiq JID
-    #
-    def lock_sync(conn)
-      return yield if locked?(conn)
+    def primed_async(conn, wait = nil, &block)
+      primed_jid = Concurrent::Promises
+                   .future(conn) { |red_con| pop_queued(red_con, wait) }
+                   .value(add_drift(wait || config.timeout))
 
-      enqueue(conn) do
-        primed_sync(conn) do
-          locked_token = call_script(:lock, key.to_a, argv, conn)
-          return yield if locked_token
-        end
-      end
+      handle_primed(primed_jid, &block)
     end
 
     #
@@ -234,10 +255,15 @@ module SidekiqUniqueJobs
     # @return [nil] when lock was not possible
     # @return [Object] whatever the block returns when lock was acquired
     #
-    def primed_sync(conn)
-      return yield if pop_queued(conn)
+    def primed_sync(conn, wait = nil, &block)
+      primed_jid = pop_queued(conn, wait)
+      handle_primed(primed_jid, &block)
+    end
+
+    def handle_primed(primed_jid)
+      return yield job_id if [job_id, "1"].include?(primed_jid)
 
-      warn_about_timeout
+      reflect(:timeout, item) unless config.wait_for_lock?
     end
 
     #
@@ -247,20 +273,24 @@ module SidekiqUniqueJobs
     #
     # @return [String] a previously enqueued token (now taken off the queue)
     #
-    def pop_queued(conn)
-      if config.wait_for_lock?
-        brpoplpush(conn)
-      else
+    def pop_queued(conn, wait = nil)
+      wait ||= config.timeout if config.wait_for_lock?
+
+      if wait.nil?
         rpoplpush(conn)
+      else
+        brpoplpush(conn, wait)
       end
     end
 
     #
     # @api private
     #
-    def brpoplpush(conn)
+    def brpoplpush(conn, wait)
+      raise InvalidArgument, "wait must be an integer" unless wait.is_a?(Integer)
+
       # passing timeout 0 to brpoplpush causes it to block indefinitely
-      conn.brpoplpush(key.queued, key.primed, timeout: config.timeout || 0)
+      conn.brpoplpush(key.queued, key.primed, timeout: wait)
     end
 
     #
@@ -270,27 +300,6 @@ module SidekiqUniqueJobs
       conn.rpoplpush(key.queued, key.primed)
     end
 
-    #
-    # Prepares all the various lock data
-    #
-    # @param [Redis] conn a redis connection
-    #
-    # @return [nil] when redis was already prepared for this lock
-    # @return [yield<String>] when successfully enqueued
-    #
-    def enqueue(conn)
-      queued_token, elapsed = timed do
-        call_script(:queue, key.to_a, argv, conn)
-      end
-
-      validity = config.pttl - elapsed - drift(config.pttl)
-
-      return unless queued_token && (validity >= 0 || config.pttl.zero?)
-
-      write_lock_info(conn)
-      yield
-    end
-
     #
     # Writes lock information to redis.
     #   The lock information contains information about worker, queue, limit etc.
@@ -299,7 +308,7 @@ module SidekiqUniqueJobs
     # @return [void]
     #
     def write_lock_info(conn)
-      return unless config.lock_info
+      return unless config.lock_info?
 
       conn.set(key.info, lock_info)
     end
@@ -315,7 +324,11 @@ module SidekiqUniqueJobs
       # Add 2 milliseconds to the drift to account for Redis expires
       # precision, which is 1 millisecond, plus 1 millisecond min drift
       # for small TTLs.
-      (val.to_i * CLOCK_DRIFT_FACTOR).to_i + 2
+      (val + 2).to_f * CLOCK_DRIFT_FACTOR
+    end
+
+    def add_drift(val)
+      val + drift(val)
     end
 
     #
@@ -329,8 +342,8 @@ module SidekiqUniqueJobs
       conn.hexists(key.locked, job_id)
     end
 
-    def warn_about_timeout
-      log_warn("Timed out after #{config.timeout}s while waiting for primed token (digest: #{key}, job_id: #{job_id})")
+    def argv
+      [job_id, config.pttl, config.type, config.limit]
     end
 
     def lock_info
@@ -340,7 +353,7 @@ module SidekiqUniqueJobs
         LIMIT => item[LOCK_LIMIT],
         TIMEOUT => item[LOCK_TIMEOUT],
         TTL => item[LOCK_TTL],
-        LOCK => config.type,
+        TYPE => config.type,
         LOCK_ARGS => item[LOCK_ARGS],
         TIME => now_f,
       )

data/lib/sidekiq_unique_jobs/logging.rb

@@ -29,8 +29,9 @@ module SidekiqUniqueJobs
     #
     # @yield [String, Exception] the message or exception to use for log message
     #
-    def log_debug(message_or_exception = nil, &block)
-      logger.debug(message_or_exception, &block)
+    def log_debug(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.debug(message, &block)
       nil
     end
 
@@ -43,8 +44,9 @@ module SidekiqUniqueJobs
     #
     # @yield [String, Exception] the message or exception to use for log message
     #
-    def log_info(message_or_exception = nil, &block)
-      logger.info(message_or_exception, &block)
+    def log_info(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.info(message, &block)
       nil
     end
 
@@ -57,8 +59,9 @@ module SidekiqUniqueJobs
     #
     # @yield [String, Exception] the message or exception to use for log message
     #
-    def log_warn(message_or_exception = nil, &block)
-      logger.warn(message_or_exception, &block)
+    def log_warn(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.warn(message, &block)
       nil
     end
 
@@ -71,8 +74,9 @@ module SidekiqUniqueJobs
     #
     # @yield [String, Exception] the message or exception to use for log message
     #
-    def log_error(message_or_exception = nil, &block)
-      logger.error(message_or_exception, &block)
+    def log_error(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.error(message, &block)
       nil
     end
 
@@ -85,11 +89,36 @@ module SidekiqUniqueJobs
     #
     # @yield [String, Exception] the message or exception to use for log message
     #
-    def log_fatal(message_or_exception = nil, &block)
-      logger.fatal(message_or_exception, &block)
+    def log_fatal(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.fatal(message, &block)
+
       nil
     end
 
+    #
+    # Build a log message
+    #
+    # @param [String, Exception] message_or_exception an entry to log
+    # @param [Hash] item the sidekiq job hash
+    #
+    # @return [String] a complete log entry
+    #
+    def build_message(message_or_exception, item = nil)
+      return nil if message_or_exception.nil?
+      return message_or_exception if item.nil?
+
+      message = message_or_exception.dup
+      details = item.slice(LOCK, QUEUE, CLASS, JID, LOCK_DIGEST).each_with_object([]) do |(key, value), memo|
+        memo << "#{key}=#{value}"
+      end
+      message << " ("
+      message << details.join(" ")
+      message << ")"
+
+      message
+    end
+
     #
     # Wraps the middleware logic with context aware logging
     #
@@ -172,7 +201,7 @@ module SidekiqUniqueJobs
     end
 
     def sidekiq_logger_context_method
-      logger.method(:with_context)           if logger_respond_to_with_context?
+      logger.method(:with_context) if logger_respond_to_with_context?
     end
 
     def sidekiq_logging_context_method

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -76,6 +76,9 @@ if pttl and pttl > 0 then
 
   log_debug("PEXPIRE", locked, pttl)
   redis.call("PEXPIRE", locked, pttl)
+
+  log_debug("PEXPIRE", info, pttl)
+  redis.call("PEXPIRE", info, pttl)
 end
 
 log_debug("PEXPIRE", queued, 1000)
@@ -84,9 +87,6 @@ redis.call("PEXPIRE", queued, 1000)
 log_debug("PEXPIRE", primed, 1000)
 redis.call("PEXPIRE", primed, 1000)
 
-log_debug("PEXPIRE", info, 1000)
-redis.call("PEXPIRE", info, 1000)
-
 log("Locked")
 log_debug("END lock digest:", digest, "job_id:", job_id)
 return job_id

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -27,7 +27,7 @@ local function find_digest_in_process_set(digest, threshold)
       else
         for i = 1, #jobs, 2 do
           local jobstr = jobs[i +1]
-          if string.find(jobstr, digest) then
+          if string.find(string.gsub(jobstr, ':RUN', ''), string.gsub(digest, ':RUN', '')) then
             log_debug("Found digest", digest, "in:", workers_key)
             found = true
             break

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -65,11 +65,6 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, -1, job_id)
 redis.call("LREM", primed, -1, job_id)
 
-if limit and limit <= 1 and locked_count and locked_count <= 1 then
-  log_debug("ZREM", digests, digest)
-  redis.call("ZREM", digests, digest)
-end
-
 local redis_version = toversion(redisversion)
 local del_cmd       = "DEL"
 
@@ -83,6 +78,18 @@ if lock_type ~= "until_expired" then
   redis.call("HDEL", locked, job_id)
 end
 
+local locked_count = redis.call("HLEN", locked)
+
+if locked_count and locked_count < 1 then
+  log_debug(del_cmd, locked)
+  redis.call(del_cmd, locked)
+end
+
+if limit and limit <= 1 and locked_count and locked_count <= 1 then
+  log_debug("ZREM", digests, digest)
+  redis.call("ZREM", digests, digest)
+end
+
 log_debug("LPUSH", queued, "1")
 redis.call("LPUSH", queued, "1")
 

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,7 +6,12 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
 
       # Calls this client middleware
       #   Used from Sidekiq.process_single
@@ -25,18 +30,11 @@ module SidekiqUniqueJobs
       private
 
       def lock
-        if (_token = lock_instance.lock)
-          yield
-        else
-          warn_about_duplicate
+        lock_instance.lock do
+          reflect(:locked, item)
+          return yield
         end
       end
-
-      def warn_about_duplicate
-        return unless log_duplicate?
-
-        log_warn "Already locked with another job_id (#{dump_json(item)})"
-      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -9,6 +9,7 @@ module SidekiqUniqueJobs
       include SidekiqUniqueJobs::SidekiqWorkerMethods
       include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::Reflectable
 
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
@@ -20,10 +21,13 @@ module SidekiqUniqueJobs
       #   This will mess up sidekiq stats because a new job is created
       def call
         if sidekiq_worker_class?
-          log_info("Rescheduling #{item[LOCK_DIGEST]}")
-          worker_class.perform_in(5, *item[ARGS])
+          if worker_class.perform_in(5, *item[ARGS])
+            reflect(:rescheduled, item)
+          else
+            reflect(:reschedule_failed, item)
+          end
         else
-          log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
+          reflect(:unknown_sidekiq_worker, item)
         end
       end
     end

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -29,15 +29,6 @@ module SidekiqUniqueJobs
       !unique_enabled?
     end
 
-    # Should duplicate payloads be logged?
-    #
-    #
-    # @return [true, false, nil]
-    #
-    def log_duplicate?
-      options[LOG_DUPLICATE] || item[LOG_DUPLICATE]
-    end
-
     #
     # A new lock for this Sidekiq Job
     #
@@ -55,10 +46,8 @@ module SidekiqUniqueJobs
     # @return [Class]
     #
     def lock_class
-      @lock_class ||= begin
-        locks.fetch(lock_type.to_sym) do
-          raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
-        end
+      @lock_class ||= locks.fetch(lock_type.to_sym) do
+        raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
       end
     end