sidekiq-unique-jobs 7.0.8 → 7.1.0

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -13,30 +13,52 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilAndWhileExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(origin: :client)
+        return lock_failed(origin: origin) unless (token = locksmith.lock)
+        return yield token if block_given?
+
+        token
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        if unlock
-          lock_on_failure do
-            runtime_lock.execute { return yield }
-          end
+        if locksmith.unlock
+          # ensure_relocked do
+          runtime_lock.execute { return yield }
+          # end
         else
-          log_warn("Couldn't unlock digest: #{item[LOCK_DIGEST]}, jid: #{item[JID]}")
+          reflect(:unlock_failed, item)
         end
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock(wait: 2)
+
+        raise
       end
 
       private
 
-      def lock_on_failure
+      def ensure_relocked
         yield
       rescue Exception # rubocop:disable Lint/RescueException
-        log_error("Runtime lock failed to execute job, restoring server lock", item)
-        lock
+        reflect(:execution_failed, item)
+        locksmith.lock
+
         raise
       end
 
       def runtime_lock
-        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
+        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -8,16 +8,28 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuted < BaseLock
-      OK ||= "OK"
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock
+        return lock_failed(origin: :client) unless (token = locksmith.lock)
+        return yield token if block_given?
+
+        token
+      end
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        lock do
+        locksmith.execute do
           yield
-          unlock_with_callback
-          callback_safely
-          item[JID]
+          unlock_and_callback
         end
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -8,10 +8,24 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      def lock
+        return lock_failed unless (job_id = locksmith.lock)
+        return yield job_id if block_given?
+
+        job_id
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        unlock_with_callback
+        callback_safely if locksmith.unlock
         yield
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -8,6 +8,27 @@ module SidekiqUniqueJobs
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     class UntilExpired < UntilExecuted
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock
+        return lock_failed unless (job_id = locksmith.lock)
+        return yield job_id if block_given?
+
+        job_id
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute(&block)
+        locksmith.execute(&block)
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -29,7 +29,10 @@ module SidekiqUniqueJobs
       #   These locks should only ever be created in the server process.
       # @return [true] always returns true
       def lock
-        true
+        job_id = item[JID]
+        yield job_id if block_given?
+
+        job_id
       end
 
       # Executes in the Sidekiq server process.
@@ -37,13 +40,13 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         with_logging_context do
-          server_strategy&.call unless locksmith.lock do
+          call_strategy(origin: :server) unless locksmith.execute do
             yield
-            callback_safely
+            callback_safely if locksmith.unlock
+          ensure
+            locksmith.unlock
           end
         end
-      ensure
-        locksmith.unlock!
       end
 
       private
@@ -51,7 +54,9 @@ module SidekiqUniqueJobs
       # This is safe as the base_lock always creates a new digest
       #   The append there for needs to be done every time
       def append_unique_key_suffix
-        item[LOCK_DIGEST] = item[LOCK_DIGEST] + RUN_SUFFIX
+        return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
+
+        item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -58,7 +58,7 @@ module SidekiqUniqueJobs
 
     def initialize(job_hash = {})
       @type        = job_hash[LOCK]&.to_sym
-      @worker      = job_hash[CLASS]
+      @worker      = SidekiqUniqueJobs.safe_constantize(job_hash[CLASS])
       @limit       = job_hash.fetch(LOCK_LIMIT, 1)
       @timeout     = job_hash.fetch(LOCK_TIMEOUT, 0)
       @ttl         = job_hash.fetch(LOCK_TTL) { job_hash.fetch(LOCK_EXPIRATION, nil) }.to_i
@@ -82,7 +82,7 @@ module SidekiqUniqueJobs
     # @return [true,fakse]
     #
     def wait_for_lock?
-      timeout.nil? || timeout.positive?
+      timeout && (timeout.zero? || timeout.positive?)
     end
 
     #

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -13,6 +13,10 @@ module SidekiqUniqueJobs
     # @!parse include SidekiqUniqueJobs::Logging
     include SidekiqUniqueJobs::Logging
 
+    # includes "SidekiqUniqueJobs::Reflectable"
+    # @!parse include SidekiqUniqueJobs::Reflectable
+    include SidekiqUniqueJobs::Reflectable
+
     # includes "SidekiqUniqueJobs::Timing"
     # @!parse include SidekiqUniqueJobs::Timing
     include SidekiqUniqueJobs::Timing
@@ -77,7 +81,7 @@ module SidekiqUniqueJobs
     # Deletes the lock regardless of if it has a pttl set
     #
     def delete!
-      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).positive?
+      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).to_i.positive?
     end
 
     #
@@ -85,16 +89,23 @@ module SidekiqUniqueJobs
     #
     # @return [String] the Sidekiq job_id that was locked/queued
     #
-    def lock(&block)
+    def lock(wait: nil)
+      method_name = wait ? :primed_async : :primed_sync
       redis(redis_pool) do |conn|
-        return lock_async(conn, &block) if block
-
-        lock_sync(conn) do
+        lock!(conn, method(method_name), wait) do
           return job_id
         end
       end
     end
 
+    def execute(&block)
+      raise SidekiqUniqueJobs::InvalidArgument, "#execute needs a block" unless block
+
+      redis(redis_pool) do |conn|
+        lock!(conn, method(:primed_async), &block)
+      end
+    end
+
     #
     # Removes the lock keys from Redis if locked by the provided jid/token
     #
@@ -114,11 +125,17 @@ module SidekiqUniqueJobs
     # @return [String] Sidekiq job_id (jid) if successful
     #
     def unlock!(conn = nil)
-      call_script(:unlock, key.to_a, argv, conn)
+      call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
+        reflect(:debug, :unlocked, item, unlocked_jid) if unlocked_jid == job_id
+
+        unlocked_jid
+      end
     end
 
     # Checks if this instance is considered locked
     #
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    #
     # @return [true, false] true when the :LOCKED hash contains the job_id
     #
     def locked?(conn = nil)
@@ -134,7 +151,7 @@ module SidekiqUniqueJobs
     # @return [String]
     #
     def to_s
-      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id}, locked=#{locked?})"
+      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id} locked=#{locked?})"
     end
 
     #
@@ -159,70 +176,71 @@ module SidekiqUniqueJobs
 
     attr_reader :redis_pool
 
-    def argv
-      [job_id, config.pttl, config.type, config.limit]
-    end
-
-    #
-    # Used for runtime locks that need automatic unlock after yielding
-    #
-    # @param [Redis] conn a redis connection
     #
-    # @return [nil] when lock was not possible
-    # @return [Object] whatever the block returns when lock was acquired
+    # Used to reduce some duplication from the two methods
     #
-    # @yieldparam [String] job_id a Sidekiq JID
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    # @param [Method] primed_method reference to the method to use for getting a primed token
     #
-    def lock_async(conn)
+    # @yieldparam [string] job_id the sidekiq JID
+    # @yieldreturn [void] whatever the calling block returns
+    def lock!(conn, primed_method, wait = nil)
       return yield job_id if locked?(conn)
 
-      enqueue(conn) do
-        primed_async(conn) do
-          locked_token = call_script(:lock, key.to_a, argv, conn)
-          return yield if locked_token == job_id
+      enqueue(conn) do |queued_jid|
+        reflect(:debug, item, queued_jid)
+
+        primed_method.call(conn, wait) do |primed_jid|
+          reflect(:debug, :primed, item, primed_jid)
+
+          locked_jid = call_script(:lock, key.to_a, argv, conn)
+          if locked_jid
+            reflect(:debug, :locked, item, locked_jid)
+            return yield job_id
+          end
         end
       end
-    ensure
-      unlock!(conn)
     end
 
     #
-    # Pops an enqueued token
-    # @note Used for runtime locks to avoid problems with blocking commands
-    #   in current thread
+    # Prepares all the various lock data
     #
     # @param [Redis] conn a redis connection
     #
-    # @return [nil] when lock was not possible
-    # @return [Object] whatever the block returns when lock was acquired
+    # @return [nil] when redis was already prepared for this lock
+    # @return [yield<String>] when successfully enqueued
     #
-    def primed_async(conn)
-      return yield if Concurrent::Promises
-                      .future(conn) { |red_con| pop_queued(red_con) }
-                      .value(drift(config.pttl) / 1000) # Important to reduce time spent waiting
+    def enqueue(conn)
+      queued_jid, elapsed = timed do
+        call_script(:queue, key.to_a, argv, conn)
+      end
+
+      return unless queued_jid
+      return unless [job_id, "1"].include?(queued_jid)
 
-      warn_about_timeout
+      validity = config.pttl - elapsed - drift(config.pttl)
+      return unless validity >= 0 || config.pttl.zero?
+
+      write_lock_info(conn)
+      yield job_id
     end
 
     #
-    # Used for non-runtime locks (no block was given)
+    # Pops an enqueued token
+    # @note Used for runtime locks to avoid problems with blocking commands
+    #   in current thread
     #
     # @param [Redis] conn a redis connection
     #
     # @return [nil] when lock was not possible
     # @return [Object] whatever the block returns when lock was acquired
     #
-    # @yieldparam [String] job_id a Sidekiq JID
-    #
-    def lock_sync(conn)
-      return yield if locked?(conn)
+    def primed_async(conn, wait = nil, &block)
+      primed_jid = Concurrent::Promises
+                   .future(conn) { |red_con| pop_queued(red_con, wait) }
+                   .value(add_drift(wait || config.ttl))
 
-      enqueue(conn) do
-        primed_sync(conn) do
-          locked_token = call_script(:lock, key.to_a, argv, conn)
-          return yield locked_token if locked_token
-        end
-      end
+      handle_primed(primed_jid, &block)
     end
 
     #
@@ -234,12 +252,15 @@ module SidekiqUniqueJobs
     # @return [nil] when lock was not possible
     # @return [Object] whatever the block returns when lock was acquired
     #
-    def primed_sync(conn)
-      if (popped_jid = pop_queued(conn))
-        return yield popped_jid
-      end
+    def primed_sync(conn, wait = nil, &block)
+      primed_jid = pop_queued(conn, wait)
+      handle_primed(primed_jid, &block)
+    end
 
-      warn_about_timeout
+    def handle_primed(primed_jid)
+      return yield job_id if [job_id, "1"].include?(primed_jid)
+
+      reflect(:timeout, item) unless config.wait_for_lock?
     end
 
     #
@@ -249,9 +270,9 @@ module SidekiqUniqueJobs
     #
     # @return [String] a previously enqueued token (now taken off the queue)
     #
-    def pop_queued(conn)
-      if config.wait_for_lock?
-        brpoplpush(conn)
+    def pop_queued(conn, wait = nil)
+      if wait || config.wait_for_lock?
+        brpoplpush(conn, wait)
       else
         rpoplpush(conn)
       end
@@ -260,9 +281,10 @@ module SidekiqUniqueJobs
     #
     # @api private
     #
-    def brpoplpush(conn)
+    def brpoplpush(conn, wait = nil)
+      wait ||= config.timeout
       # passing timeout 0 to brpoplpush causes it to block indefinitely
-      conn.brpoplpush(key.queued, key.primed, timeout: config.timeout)
+      conn.brpoplpush(key.queued, key.primed, timeout: wait)
     end
 
     #
@@ -272,27 +294,6 @@ module SidekiqUniqueJobs
       conn.rpoplpush(key.queued, key.primed)
     end
 
-    #
-    # Prepares all the various lock data
-    #
-    # @param [Redis] conn a redis connection
-    #
-    # @return [nil] when redis was already prepared for this lock
-    # @return [yield<String>] when successfully enqueued
-    #
-    def enqueue(conn)
-      queued_token, elapsed = timed do
-        call_script(:queue, key.to_a, argv, conn)
-      end
-
-      validity = config.pttl - elapsed - drift(config.pttl)
-
-      return unless queued_token && (validity >= 0 || config.pttl.zero?)
-
-      write_lock_info(conn)
-      yield queued_token
-    end
-
     #
     # Writes lock information to redis.
     #   The lock information contains information about worker, queue, limit etc.
@@ -317,7 +318,11 @@ module SidekiqUniqueJobs
       # Add 2 milliseconds to the drift to account for Redis expires
       # precision, which is 1 millisecond, plus 1 millisecond min drift
       # for small TTLs.
-      (val.to_f * CLOCK_DRIFT_FACTOR).to_f + 2
+      (val + 2).to_f * CLOCK_DRIFT_FACTOR
+    end
+
+    def add_drift(val)
+      val + drift(val)
     end
 
     #
@@ -331,8 +336,8 @@ module SidekiqUniqueJobs
       conn.hexists(key.locked, job_id)
     end
 
-    def warn_about_timeout
-      log_warn("Timed out after #{config.timeout}s while waiting for primed token (digest: #{key}, job_id: #{job_id})")
+    def argv
+      [job_id, config.pttl, config.type, config.limit]
     end
 
     def lock_info
@@ -342,7 +347,7 @@ module SidekiqUniqueJobs
         LIMIT => item[LOCK_LIMIT],
         TIMEOUT => item[LOCK_TIMEOUT],
         TTL => item[LOCK_TTL],
-        LOCK => config.type,
+        TYPE => config.type,
         LOCK_ARGS => item[LOCK_ARGS],
         TIME => now_f,
       )