sidekiq-unique-jobs 6.0.6 → 6.0.25

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -13,9 +13,10 @@ module SidekiqUniqueJobs
       # @param [Proc] callback the callback to use after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
       def initialize(item, callback, redis_pool = nil)
-        @item       = prepare_item(item)
+        @item       = item
         @callback   = callback
         @redis_pool = redis_pool
+        add_uniqueness_when_missing # Used to ease testing
       end
 
       # Handles locking of sidekiq jobs.
@@ -65,6 +66,14 @@ module SidekiqUniqueJobs
 
       private
 
+      def add_uniqueness_when_missing
+        return if item.key?(UNIQUE_DIGEST_KEY)
+
+        # The below should only be done to ease testing
+        # in production this will be done by the middleware
+        SidekiqUniqueJobs::Job.add_uniqueness(item)
+      end
+
       def call_strategy
         @attempt += 1
         strategy.call { lock if replace? }
@@ -99,36 +108,23 @@ module SidekiqUniqueJobs
       def with_cleanup
         yield
       rescue Sidekiq::Shutdown
-        notify_about_manual_unlock
+        log_info("Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!")
         raise
       else
         unlock_with_callback
       end
 
-      def prepare_item(item)
-        calculator = SidekiqUniqueJobs::Timeout::Calculator.new(item)
-        item[LOCK_TIMEOUT_KEY]    = calculator.lock_timeout
-        item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
-        SidekiqUniqueJobs::UniqueArgs.digest(item)
-        item
-      end
-
-      def notify_about_manual_unlock
-        log_fatal("the unique_key: #{item[UNIQUE_DIGEST_KEY]} needs to be unlocked manually")
-        false
-      end
-
       def unlock_with_callback
-        return notify_about_manual_unlock unless unlock
+        return log_warn("might need to be unlocked manually") unless unlock
 
         callback_safely
-        item[JID_KEY]
       end
 
       def callback_safely
         callback&.call
+        item[JID_KEY]
       rescue StandardError
-        log_warn("The unique_key: #{item[UNIQUE_DIGEST_KEY]} has been unlocked but the #after_unlock callback failed!")
+        log_warn("unlocked successfully but the #after_unlock callback failed!")
         raise
       end
 

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -16,15 +16,31 @@ module SidekiqUniqueJobs
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        return unless locked?
-        unlock
-
-        runtime_lock.execute { yield }
+        if unlock
+          lock_on_failure do
+            runtime_lock.execute { return yield }
+          end
+        else
+          log_warn "couldn't unlock digest: #{item[UNIQUE_DIGEST_KEY]} #{item[JID_KEY]}"
+        end
+      ensure
+        runtime_lock.delete!
       end
 
       def runtime_lock
         @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
       end
+
+      private
+
+      def lock_on_failure
+        yield
+        runtime_lock.delete!
+      rescue Exception # rubocop:disable Lint/RescueException
+        log_error("Failed to execute job, restoring lock")
+        lock
+        raise
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -8,13 +8,17 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@zoolutions.se>
     class UntilExecuted < BaseLock
-      OK ||= 'OK'
+      OK ||= "OK"
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        return unless locked?
-        with_cleanup { yield }
+        if locked?
+          with_cleanup { yield }
+        else
+          log_warn "the unique_key: #{item[UNIQUE_DIGEST_KEY]} is not locked, allowing job to silently complete"
+          nil
+        end
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   class Lock
     # Locks jobs until {#execute} starts
     # - Locks on perform_in or perform_async
-    # - Unlocks after yielding to the worker's perform method
+    # - Unlocks before yielding to the worker's perform method
     #
     # @author Mikael Henriksson <mikael@zoolutions.se>
     class UntilExecuting < BaseLock

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -21,6 +21,7 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         return unless locked?
+
         yield
         # this lock does not handle after_unlock since we don't know when that would happen
       end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -11,11 +11,12 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@zoolutions.se>
     class WhileExecuting < BaseLock
-      RUN_SUFFIX ||= ':RUN'
+      RUN_SUFFIX ||= ":RUN"
 
       # @param [Hash] item the Sidekiq job hash
       # @param [Proc] callback callback to call after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+      #
       def initialize(item, callback, redis_pool = nil)
         super(item, callback, redis_pool)
         append_unique_key_suffix
@@ -32,8 +33,13 @@ module SidekiqUniqueJobs
       #   These jobs are locked in the server process not from the client
       # @yield to the worker class perform method
       def execute
-        return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
-        with_cleanup { yield }
+        return strategy&.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
+
+        yield
+        unlock_with_callback
+      rescue Exception # rubocop:disable Lint/RescueException
+        delete!
+        raise
       end
 
       private

data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb

@@ -11,14 +11,6 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@zoolutions.se>
     class WhileExecutingReject < WhileExecuting
-      # Executes in the Sidekiq server process
-      # @yield to the worker class perform method
-      def execute
-        return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
-
-        with_cleanup { yield }
-      end
-
       # Overridden with a forced {OnConflict::Reject} strategy
       # @return [OnConflict::Reject] a reject strategy
       def strategy

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -4,6 +4,7 @@ module SidekiqUniqueJobs
   # Lock manager class that handles all the various locks
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
+  # rubocop:disable Metrics/ClassLength
   class Locksmith
     include SidekiqUniqueJobs::Connection
 
@@ -13,34 +14,26 @@ module SidekiqUniqueJobs
     # @option item [String] :unique_digest the unique digest (See: {UniqueArgs#unique_digest})
     # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
     def initialize(item, redis_pool = nil)
-      @concurrency   = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
-      @expiration    = item[LOCK_EXPIRATION_KEY]
+      # @concurrency   = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
+      @ttl           = item[LOCK_EXPIRATION_KEY] || item[LOCK_TTL_KEY]
       @jid           = item[JID_KEY]
-      @unique_digest = item[UNIQUE_DIGEST_KEY]
+      @unique_digest = item[UNIQUE_DIGEST_KEY] || item[LOCK_DIGEST_KEY]
+      @lock_type     = item[LOCK_KEY] || item[UNIQUE_KEY]
+      @lock_type   &&= @lock_type.to_sym
       @redis_pool    = redis_pool
     end
 
-    # Checks if the exists key is created in redis
-    # @return [true, false]
-    def exists?
-      redis(redis_pool) { |conn| conn.exists(exists_key) }
-    end
-
-    # The number of available resourced for this lock
-    # @return [Integer] the number of available resources
-    def available_count
-      return concurrency unless exists?
-
-      redis(redis_pool) { |conn| conn.llen(available_key) }
-    end
-
-    # Deletes the lock unless it has an expiration set
+    #
+    # Deletes the lock unless it has a ttl set
+    #
+    #
     def delete
-      return if expiration
+      return if ttl
+
       delete!
     end
 
-    # Deletes the lock regardless of if it has an expiration set
+    # Deletes the lock regardless of if it has a ttl set
     def delete!
       Scripts.call(
         :delete,
@@ -49,15 +42,18 @@ module SidekiqUniqueJobs
       )
     end
 
+    #
     # Create a lock for the item
+    #
     # @param [Integer] timeout the number of seconds to wait for a lock.
-    #   nil means wait indefinitely
-    # @yield the block to execute if a lock is successful
-    # @return the Sidekiq job_id (jid)
+    #
+    # @return [String] the Sidekiq job_id (jid)
+    #
+    #
     def lock(timeout = nil, &block)
       Scripts.call(:lock, redis_pool,
                    keys: [exists_key, grabbed_key, available_key, UNIQUE_SET, unique_digest],
-                   argv: [jid, expiration])
+                   argv: [jid, ttl, lock_type])
 
       grab_token(timeout) do |token|
         touch_grabbed_token(token)
@@ -66,18 +62,27 @@ module SidekiqUniqueJobs
     end
     alias wait lock
 
+    #
     # Removes the lock keys from Redis if locked by the provided jid/token
+    #
     # @return [false] unless locked?
     # @return [String] Sidekiq job_id (jid) if successful
+    #
     def unlock(token = nil)
       token ||= jid
       return false unless locked?(token)
+
       unlock!(token)
     end
 
+    #
     # Removes the lock keys from Redis
+    #
+    # @param [String] token the token to unlock (defaults to jid)
+    #
     # @return [false] unless locked?
     # @return [String] Sidekiq job_id (jid) if successful
+    #
     def unlock!(token = nil)
       token ||= jid
 
@@ -85,22 +90,40 @@ module SidekiqUniqueJobs
         :unlock,
         redis_pool,
         keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
-        argv: [token, expiration],
+        argv: [token, ttl, lock_type],
       )
     end
 
-    # Checks if this instance is considered locked
+    #
     # @param [String] token the unique token to check for a lock.
     #   nil will default to the jid provided in the initializer
     # @return [true, false]
+    #
+    # Checks if this instance is considered locked
+    #
+    # @param [<type>] token <description>
+    #
+    # @return [<type>] <description>
+    #
     def locked?(token = nil)
       token ||= jid
+
+      convert_legacy_lock(token)
       redis(redis_pool) { |conn| conn.hexists(grabbed_key, token) }
     end
 
     private
 
-    attr_reader :concurrency, :unique_digest, :expiration, :jid, :redis_pool
+    attr_reader :unique_digest, :ttl, :jid, :redis_pool, :lock_type
+
+    def convert_legacy_lock(token)
+      Scripts.call(
+        :convert_legacy_lock,
+        redis_pool,
+        keys: [grabbed_key, unique_digest],
+        argv: [token, current_time.to_f],
+      )
+    end
 
     def grab_token(timeout = nil)
       redis(redis_pool) do |conn|
@@ -116,7 +139,10 @@ module SidekiqUniqueJobs
     end
 
     def touch_grabbed_token(token)
-      redis(redis_pool) { |conn| conn.hset(grabbed_key, token, current_time.to_f) }
+      redis(redis_pool) do |conn|
+        conn.hset(grabbed_key, token, current_time.to_f)
+        conn.expire(grabbed_key, ttl) if ttl && lock_type == :until_expired
+      end
     end
 
     def return_token_or_block_value(token)
@@ -131,19 +157,19 @@ module SidekiqUniqueJobs
     end
 
     def available_key
-      @available_key ||= namespaced_key('AVAILABLE')
+      @available_key ||= namespaced_key("AVAILABLE")
     end
 
     def exists_key
-      @exists_key ||= namespaced_key('EXISTS')
+      @exists_key ||= namespaced_key("EXISTS")
     end
 
     def grabbed_key
-      @grabbed_key ||= namespaced_key('GRABBED')
+      @grabbed_key ||= namespaced_key("GRABBED")
     end
 
     def version_key
-      @version_key ||= namespaced_key('VERSION')
+      @version_key ||= namespaced_key("VERSION")
     end
 
     def namespaced_key(variable)
@@ -159,4 +185,5 @@ module SidekiqUniqueJobs
       redis(&:time)
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/sidekiq_unique_jobs/logging.rb

@@ -16,6 +16,7 @@ module SidekiqUniqueJobs
     #   Used for compatibility with logger
     def log_debug(message_or_exception = nil, &block)
       logger.debug(message_or_exception, &block)
+      nil
     end
 
     # Logs a message at info level
@@ -24,6 +25,7 @@ module SidekiqUniqueJobs
     #   Used for compatibility with logger
     def log_info(message_or_exception = nil, &block)
       logger.info(message_or_exception, &block)
+      nil
     end
 
     # Logs a message at warn level
@@ -32,6 +34,7 @@ module SidekiqUniqueJobs
     #   Used for compatibility with logger
     def log_warn(message_or_exception = nil, &block)
       logger.warn(message_or_exception, &block)
+      nil
     end
 
     # Logs a message at error level
@@ -40,6 +43,7 @@ module SidekiqUniqueJobs
     #   Used for compatibility with logger
     def log_error(message_or_exception = nil, &block)
       logger.error(message_or_exception, &block)
+      nil
     end
 
     # Logs a message at fatal level
@@ -48,6 +52,16 @@ module SidekiqUniqueJobs
     #   Used for compatibility with logger
     def log_fatal(message_or_exception = nil, &block)
       logger.fatal(message_or_exception, &block)
+      nil
+    end
+
+    def logging_context(middleware_class, job_hash)
+      digest = job_hash["unique_digest"]
+      if defined?(Sidekiq::Logging)
+        "#{middleware_class} #{"DIG-#{digest}" if digest}"
+      else
+        { middleware: middleware_class, unique_digest: digest }
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -1,8 +1,13 @@
 # frozen_string_literal: true
 
-require 'sidekiq'
+require "sidekiq"
 
 module SidekiqUniqueJobs
+  #
+  # Provides the sidekiq middleware that makes the gem work
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
   module Middleware
     def self.extended(base)
       base.class_eval do
@@ -15,16 +20,24 @@ module SidekiqUniqueJobs
       configure_client_middleware
     end
 
-    def configure_server_middleware
+    def configure_server_middleware # rubocop:disable Metrics/MethodLength
       Sidekiq.configure_server do |config|
         config.client_middleware do |chain|
-          require 'sidekiq_unique_jobs/client/middleware'
-          chain.add SidekiqUniqueJobs::Client::Middleware
+          require "sidekiq_unique_jobs/client/middleware"
+          if defined?(Apartment::Sidekiq::Middleware::Client)
+            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Client::Middleware
+          else
+            chain.add SidekiqUniqueJobs::Client::Middleware
+          end
         end
 
         config.server_middleware do |chain|
-          require 'sidekiq_unique_jobs/server/middleware'
-          chain.add SidekiqUniqueJobs::Server::Middleware
+          require "sidekiq_unique_jobs/server/middleware"
+          if defined?(Apartment::Sidekiq::Middleware::Server)
+            chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Server::Middleware
+          else
+            chain.add SidekiqUniqueJobs::Server::Middleware
+          end
         end
       end
     end
@@ -32,11 +45,15 @@ module SidekiqUniqueJobs
     def configure_client_middleware
       Sidekiq.configure_client do |config|
         config.client_middleware do |chain|
-          require 'sidekiq_unique_jobs/client/middleware'
-          chain.add SidekiqUniqueJobs::Client::Middleware
+          require "sidekiq_unique_jobs/client/middleware"
+          if defined?(Apartment::Sidekiq::Middleware::Client)
+            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Client::Middleware
+          else
+            chain.add SidekiqUniqueJobs::Client::Middleware
+          end
         end
       end
     end
   end
 end
-SidekiqUniqueJobs.send(:extend, SidekiqUniqueJobs::Middleware)
+SidekiqUniqueJobs.extend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/normalizer.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
 module SidekiqUniqueJobs
   # Normalizes hashes by dumping them to json and loading them from json

data/lib/sidekiq_unique_jobs/on_conflict/raise.rb

@@ -10,7 +10,7 @@ module SidekiqUniqueJobs
       #   This will cause Sidekiq to retry the job
       # @raise [SidekiqUniqueJobs::Conflict]
       def call
-        fail SidekiqUniqueJobs::Conflict, item
+        raise SidekiqUniqueJobs::Conflict, item
       end
     end
   end

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -53,9 +53,9 @@ module SidekiqUniqueJobs
       def push_to_deadset
         Sidekiq.redis do |conn|
           conn.multi do
-            conn.zadd('dead', current_time, payload)
-            conn.zremrangebyscore('dead', '-inf', current_time - Sidekiq::DeadSet.timeout)
-            conn.zremrangebyrank('dead', 0, -Sidekiq::DeadSet.max_jobs)
+            conn.zadd("dead", current_time, payload)
+            conn.zremrangebyscore("dead", "-inf", current_time - Sidekiq::DeadSet.timeout)
+            conn.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
           end
         end
       end

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -19,18 +19,23 @@ module SidekiqUniqueJobs
       # @yield to retry the lock after deleting the old one
       def call(&block)
         return unless delete_job_by_digest
+
         delete_lock
         block&.call
       end
 
       # Delete the job from either schedule, retry or the queue
       def delete_job_by_digest
-        Scripts.call(:delete_job_by_digest, nil, keys: [queue, unique_digest])
+        Scripts.call(
+          :delete_job_by_digest,
+          nil,
+          keys: ["#{QUEUE_KEY}:#{queue}", SCHEDULE_SET, RETRY_SET], argv: [unique_digest],
+        )
       end
 
       # Delete the keys belonging to the job
       def delete_lock
-        Scripts.call(:delete_by_digest, nil, keys: [UNIQUE_SET, unique_digest])
+        SidekiqUniqueJobs::Digests.delete_by_digest(unique_digest)
       end
     end
   end

data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
       # Use strategy on conflict
       # @raise [NotImplementedError] needs to be implemented in child class
       def call
-        fail NotImplementedError, 'needs to be implemented in child class'
+        raise NotImplementedError, "needs to be implemented in child class"
       end
 
       def replace?

data/lib/sidekiq_unique_jobs/on_conflict.rb

@@ -1,14 +1,19 @@
 # frozen_string_literal: true
 
-require_relative 'on_conflict/strategy'
-require_relative 'on_conflict/null_strategy'
-require_relative 'on_conflict/log'
-require_relative 'on_conflict/raise'
-require_relative 'on_conflict/reject'
-require_relative 'on_conflict/replace'
-require_relative 'on_conflict/reschedule'
+require_relative "on_conflict/strategy"
+require_relative "on_conflict/null_strategy"
+require_relative "on_conflict/log"
+require_relative "on_conflict/raise"
+require_relative "on_conflict/reject"
+require_relative "on_conflict/replace"
+require_relative "on_conflict/reschedule"
 
 module SidekiqUniqueJobs
+  #
+  # Provides lock conflict resolutions
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
   module OnConflict
     STRATEGIES = {
       log: OnConflict::Log,

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -11,12 +11,12 @@ module SidekiqUniqueJobs
   module OptionsWithFallback
     LOCKS = {
       until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
-      until_executed:            SidekiqUniqueJobs::Lock::UntilExecuted,
-      until_executing:           SidekiqUniqueJobs::Lock::UntilExecuting,
-      until_expired:             SidekiqUniqueJobs::Lock::UntilExpired,
-      until_timeout:             SidekiqUniqueJobs::Lock::UntilExpired,
-      while_executing:           SidekiqUniqueJobs::Lock::WhileExecuting,
-      while_executing_reject:    SidekiqUniqueJobs::Lock::WhileExecutingReject,
+      until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+      until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+      until_timeout: SidekiqUniqueJobs::Lock::UntilExpired,
+      while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
     }.freeze
 
     def self.included(base)
@@ -50,14 +50,14 @@ module SidekiqUniqueJobs
     def lock_class
       @lock_class ||= begin
         LOCKS.fetch(lock_type.to_sym) do
-          fail UnknownLock, "No implementation for `lock: :#{lock_type}`"
+          raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
         end
       end
     end
 
     # @return [Symbol]
     def lock_type
-      @lock_type ||= options[LOCK_KEY] || item[LOCK_KEY] || unique_type
+      @lock_type ||= item[LOCK_KEY] || options[LOCK_KEY] || unique_type
     end
 
     def unique_type