sidekiq-unique-jobs 6.0.25 → 7.1.5

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -5,10 +5,33 @@ module SidekiqUniqueJobs
     # Abstract base class for locks
     #
     # @abstract
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class BaseLock
+      extend Forwardable
+
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::Logging
 
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
+
+      #
+      # Validates that the sidekiq_options for the worker is valid
+      #
+      # @param [Hash] options the sidekiq_options given to the worker
+      #
+      # @return [void]
+      #
+      def self.validate_options(options = {})
+        Validator.validate(options)
+      end
+
+      # NOTE: Mainly used for a clean testing API
+      #
+      def_delegators :locksmith, :locked?
+
       # @param [Hash] item the Sidekiq job hash
       # @param [Proc] callback the callback to use after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
@@ -16,21 +39,22 @@ module SidekiqUniqueJobs
         @item       = item
         @callback   = callback
         @redis_pool = redis_pool
-        add_uniqueness_when_missing # Used to ease testing
+        @attempt    = 0
+        prepare_item # Used to ease testing
+        @lock_config = LockConfig.new(item)
       end
 
-      # Handles locking of sidekiq jobs.
-      #   Will call a conflict strategy if lock can't be achieved.
-      # @return [String] the sidekiq job id
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
       def lock
-        @attempt = 0
-        return item[JID_KEY] if locked?
-
-        if (token = locksmith.lock(item[LOCK_TIMEOUT_KEY]))
-          token
-        else
-          call_strategy
-        end
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
       end
 
       # Execute the job in the Sidekiq server processor
@@ -39,97 +63,94 @@ module SidekiqUniqueJobs
         raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
       end
 
-      # Unlocks the job from redis
-      # @return [String] sidekiq job id when successful
-      # @return [false] when unsuccessful
-      def unlock
-        locksmith.unlock(item[JID_KEY]) # Only signal to release the lock
-      end
-
-      # Deletes the job from redis if it is locked.
-      def delete
-        locksmith.delete # Soft delete (don't forcefully remove when expiration is set)
-      end
-
-      # Forcefully deletes the job from redis.
-      #   This is good for jobs when a previous lock was not unlocked
-      def delete!
-        locksmith.delete! # Force delete the lock
-      end
-
-      # Checks if the item has achieved a lock
-      # @return [true] when this jid has locked the job
-      # @return [false] when this jid has not locked the job
-      def locked?
-        locksmith.locked?(item[JID_KEY])
+      #
+      # The lock manager/client
+      #
+      # @api private
+      # @return [SidekiqUniqueJobs::Locksmith] the locksmith for this sidekiq job
+      #
+      def locksmith
+        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
       end
 
       private
 
-      def add_uniqueness_when_missing
-        return if item.key?(UNIQUE_DIGEST_KEY)
+      # @!attribute [r] item
+      #   @return [Hash<String, Object>] the Sidekiq job hash
+      attr_reader :item
+      # @!attribute [r] lock_config
+      #   @return [LockConfig] a lock configuration
+      attr_reader :lock_config
+      # @!attribute [r] redis_pool
+      #   @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      attr_reader :redis_pool
+      # @!attribute [r] callback
+      #   @return [Proc] the block to call after unlock
+      attr_reader :callback
+      # @!attribute [r] attempt
+      #   @return [Integer] the current locking attempt
+      attr_reader :attempt
+
+      def prepare_item
+        return if item.key?(LOCK_DIGEST)
 
         # The below should only be done to ease testing
         # in production this will be done by the middleware
-        SidekiqUniqueJobs::Job.add_uniqueness(item)
+        SidekiqUniqueJobs::Job.prepare(item)
       end
 
-      def call_strategy
-        @attempt += 1
-        strategy.call { lock if replace? }
+      #
+      # Handle when lock failed
+      #
+      # @param [Symbol] location: :client or :server
+      #
+      # @return [void]
+      #
+      def lock_failed(origin: :client)
+        reflect(:lock_failed, item)
+        call_strategy(origin: origin)
       end
 
-      def replace?
-        strategy.replace? && attempt < 2
-      end
-
-      # The sidekiq job hash
-      # @return [Hash] the Sidekiq job hash
-      attr_reader :item
-
-      # The sidekiq redis pool
-      # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
-      attr_reader :redis_pool
-
-      # The sidekiq job hash
-      # @return [Proc] the callback to use after unlock
-      attr_reader :callback
-
-      # The current attempt to lock the job
-      # @return [Integer] the numerical value of the attempt
-      attr_reader :attempt
+      def call_strategy(origin:)
+        @attempt += 1
 
-      # The interface to the locking mechanism
-      # @return [SidekiqUniqueJobs::Locksmith]
-      def locksmith
-        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
+        case origin
+        when :client
+          client_strategy.call { lock if replace? }
+        when :server
+          server_strategy.call { lock if replace? }
+        else
+          raise SidekiqUniqueJobs::InvalidArgument,
+                "either `for: :server` or `for: :client` needs to be specified"
+        end
       end
 
-      def with_cleanup
-        yield
-      rescue Sidekiq::Shutdown
-        log_info("Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!")
-        raise
-      else
-        unlock_with_callback
+      def replace?
+        client_strategy.replace? && attempt < 2
       end
 
-      def unlock_with_callback
-        return log_warn("might need to be unlocked manually") unless unlock
+      def unlock_and_callback
+        return callback_safely if locksmith.unlock
 
-        callback_safely
+        reflect(:unlock_failed, item)
       end
 
       def callback_safely
         callback&.call
-        item[JID_KEY]
+        item[JID]
       rescue StandardError
-        log_warn("unlocked successfully but the #after_unlock callback failed!")
+        reflect(:after_unlock_callback_failed, item)
         raise
       end
 
-      def strategy
-        @strategy ||= OnConflict.find_strategy(item[ON_CONFLICT_KEY]).new(item)
+      def client_strategy
+        @client_strategy ||=
+          OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)
+      end
+
+      def server_strategy
+        @server_strategy ||=
+          OnConflict.find_strategy(lock_config.on_server_conflict).new(item, redis_pool)
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/client_validator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq client process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ClientValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:raise, :reject, :reschedule].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq client process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_client_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_client_conflict] = "#{on_conflict} is incompatible with the client process"
+        lock_config
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/server_validator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq server process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ServerValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:replace].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq server process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_server_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_server_conflict] = "#{on_conflict} is incompatible with the server process"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -11,36 +11,55 @@ module SidekiqUniqueJobs
     # See {#lock} for more information about the client.
     # See {#execute} for more information about the server
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilAndWhileExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(origin: :client)
+        return lock_failed(origin: origin) unless (token = locksmith.lock)
+        return yield token if block_given?
+
+        token
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        if unlock
-          lock_on_failure do
-            runtime_lock.execute { return yield }
-          end
+        if locksmith.unlock
+          # ensure_relocked do
+          runtime_lock.execute { return yield }
+          # end
         else
-          log_warn "couldn't unlock digest: #{item[UNIQUE_DIGEST_KEY]} #{item[JID_KEY]}"
+          reflect(:unlock_failed, item)
         end
-      ensure
-        runtime_lock.delete!
-      end
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock(wait: 2)
 
-      def runtime_lock
-        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
+        raise
       end
 
       private
 
-      def lock_on_failure
+      def ensure_relocked
         yield
-        runtime_lock.delete!
       rescue Exception # rubocop:disable Lint/RescueException
-        log_error("Failed to execute job, restoring lock")
-        lock
+        reflect(:execution_failed, item)
+        locksmith.lock
+
         raise
       end
+
+      def runtime_lock
+        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -6,18 +6,30 @@ module SidekiqUniqueJobs
     # - Locks on perform_in or perform_async
     # - Unlocks after yielding to the worker's perform method
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuted < BaseLock
-      OK ||= "OK"
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock
+        return lock_failed(origin: :client) unless (token = locksmith.lock)
+        return yield token if block_given?
+
+        token
+      end
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        if locked?
-          with_cleanup { yield }
-        else
-          log_warn "the unique_key: #{item[UNIQUE_DIGEST_KEY]} is not locked, allowing job to silently complete"
-          nil
+        locksmith.execute do
+          yield
+          unlock_and_callback
         end
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -6,12 +6,26 @@ module SidekiqUniqueJobs
     # - Locks on perform_in or perform_async
     # - Unlocks before yielding to the worker's perform method
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      def lock
+        return lock_failed unless (job_id = locksmith.lock)
+        return yield job_id if block_given?
+
+        job_id
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        unlock_with_callback
+        callback_safely if locksmith.unlock
         yield
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -2,28 +2,32 @@
 
 module SidekiqUniqueJobs
   class Lock
-    # Locks jobs until the lock has expired
-    # - Locks on perform_in or perform_async
-    # - Unlocks when the expiration is hit
     #
-    # See {#lock} for more information about the client.
-    # See {#execute} for more information about the server
+    # UntilExpired locks until the job expires
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
-    class UntilExpired < BaseLock
-      # Prevents these locks from being unlocked
-      # @return [true] always returns true
-      def unlock
-        true
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class UntilExpired < UntilExecuted
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock
+        return lock_failed unless (job_id = locksmith.lock)
+        return yield job_id if block_given?
+
+        job_id
       end
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
-      def execute
-        return unless locked?
-
-        yield
-        # this lock does not handle after_unlock since we don't know when that would happen
+      def execute(&block)
+        locksmith.execute(&block)
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/validator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validator base class to avoid some duplication
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Validator
+      #
+      # @return [Hash] a hash mapping of deprecated keys and their new value
+      DEPRECATED_KEYS = {
+        UNIQUE.to_sym => LOCK.to_sym,
+        UNIQUE_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
+        LOCK_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
+        UNIQUE_PREFIX.to_sym => LOCK_PREFIX.to_sym,
+      }.freeze
+
+      #
+      # Shorthand for `new(options).validate`
+      #
+      # @param [Hash] options the sidekiq_options for the worker being validated
+      #
+      # @return [LockConfig] the lock configuration with errors if any
+      #
+      def self.validate(options)
+        new(options).validate
+      end
+
+      #
+      # @!attribute [r] lock_config
+      #   @return [LockConfig] the lock configuration for this worker
+      attr_reader :lock_config
+
+      #
+      # Initialize a new validator
+      #
+      # @param [Hash] options the sidekiq_options for the worker being validated
+      #
+      def initialize(options)
+        @options     = options.transform_keys(&:to_sym)
+        @lock_config = LockConfig.new(options)
+        handle_deprecations
+      end
+
+      #
+      # Validate the workers lock configuration
+      #
+      #
+      # @return [LockConfig] the lock configuration with errors if any
+      #
+      def validate
+        case lock_config.type
+        when :while_executing
+          validate_server
+        when :until_executing
+          validate_client
+        else
+          validate_client
+          validate_server
+        end
+
+        lock_config
+      end
+
+      #
+      # Validate deprecated keys
+      #   adds useful information about how to proceed with fixing handle_deprecations
+      #
+      # @return [void]
+      #
+      def handle_deprecations
+        DEPRECATED_KEYS.each do |old, new|
+          next unless @options.key?(old)
+
+          lock_config.errors[old] = "is deprecated, use `#{new}: #{@options[old]}` instead."
+        end
+      end
+
+      #
+      # Validates the client configuration
+      #
+      def validate_client
+        ClientValidator.validate(lock_config)
+      end
+
+      #
+      # Validates the server configuration
+      #
+      def validate_server
+        ServerValidator.validate(lock_config)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -9,10 +9,13 @@ module SidekiqUniqueJobs
     # See {#lock} for more information about the client.
     # See {#execute} for more information about the server
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class WhileExecuting < BaseLock
       RUN_SUFFIX ||= ":RUN"
 
+      include SidekiqUniqueJobs::OptionsWithFallback
+      include SidekiqUniqueJobs::Logging::Middleware
+
       # @param [Hash] item the Sidekiq job hash
       # @param [Proc] callback callback to call after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
@@ -26,20 +29,24 @@ module SidekiqUniqueJobs
       #   These locks should only ever be created in the server process.
       # @return [true] always returns true
       def lock
-        true
+        job_id = item[JID]
+        yield job_id if block_given?
+
+        job_id
       end
 
       # Executes in the Sidekiq server process.
       #   These jobs are locked in the server process not from the client
       # @yield to the worker class perform method
       def execute
-        return strategy&.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
-
-        yield
-        unlock_with_callback
-      rescue Exception # rubocop:disable Lint/RescueException
-        delete!
-        raise
+        with_logging_context do
+          call_strategy(origin: :server) unless locksmith.execute do
+            yield
+            callback_safely if locksmith.unlock
+          ensure
+            locksmith.unlock
+          end
+        end
       end
 
       private
@@ -47,7 +54,9 @@ module SidekiqUniqueJobs
       # This is safe as the base_lock always creates a new digest
       #   The append there for needs to be done every time
       def append_unique_key_suffix
-        item[UNIQUE_DIGEST_KEY] = item[UNIQUE_DIGEST_KEY] + RUN_SUFFIX
+        return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
+
+        item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb

@@ -9,12 +9,12 @@ module SidekiqUniqueJobs
     # See {#lock} for more information about the client.
     # See {#execute} for more information about the server
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class WhileExecutingReject < WhileExecuting
       # Overridden with a forced {OnConflict::Reject} strategy
       # @return [OnConflict::Reject] a reject strategy
-      def strategy
-        @strategy ||= OnConflict.find_strategy(:reject).new(item)
+      def server_strategy
+        @server_strategy ||= OnConflict.find_strategy(:reject).new(item, redis_pool)
       end
     end
   end