sidekiq-unique-jobs 6.0.25 → 7.1.29

data/lib/sidekiq_unique_jobs/json.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles loading and dumping of json
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module JSON
+    module_function
+
+    #
+    # Parses a JSON string into an object
+    #
+    # @param [String] string the object to parse
+    #
+    # @return [Object]
+    #
+    def load_json(string)
+      return if string.nil? || string.empty?
+
+      ::JSON.parse(string)
+    end
+
+    #
+    # Prevents trying JSON.load from raising errors given argument is a hash
+    #
+    # @param [String, Hash] string the JSON string to parse
+    #
+    # @return [Hash,Array]
+    #
+    def safe_load_json(string)
+      return string if string.is_a?(Hash)
+
+      load_json(string)
+    end
+
+    #
+    # Dumps an object into a JSON string
+    #
+    # @param [Object] object a JSON convertible object
+    #
+    # @return [String] a JSON string
+    #
+    def dump_json(object)
+      ::JSON.generate(object)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/key.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Key class wraps logic dealing with various lock keys
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Key
+    #
+    # @!attribute [r] digest
+    #   @return [String] the digest key for which keys are created
+    attr_reader :digest
+    #
+    # @!attribute [r] queued
+    #   @return [String] the list key with queued job_id's
+    attr_reader :queued
+    #
+    # @!attribute [r] primed
+    #   @return [String] the list key with primed job_id's
+    attr_reader :primed
+    #
+    # @!attribute [r] locked
+    #   @return [String] the hash key with locked job_id's
+    attr_reader :locked
+    #
+    # @!attribute [r] info
+    #   @return [String] information about the lock
+    attr_reader :info
+    #
+    # @!attribute [r] changelog
+    #   @return [String] the zset with changelog entries
+    attr_reader :changelog
+    #
+    # @!attribute [r] digests
+    #   @return [String] the zset with locked digests
+    attr_reader :digests
+    #
+    # @!attribute [r] expiring_digests
+    #   @return [String] the zset with locked expiring_digests
+    attr_reader :expiring_digests
+
+    #
+    # Initialize a new Key
+    #
+    # @param [String] digest the digest to use as key
+    #
+    def initialize(digest)
+      @digest           = digest
+      @queued           = suffixed_key("QUEUED")
+      @primed           = suffixed_key("PRIMED")
+      @locked           = suffixed_key("LOCKED")
+      @info             = suffixed_key("INFO")
+      @changelog        = CHANGELOGS
+      @digests          = DIGESTS
+      @expiring_digests = EXPIRING_DIGESTS
+    end
+
+    #
+    # Provides the only important information about this keys
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      digest
+    end
+
+    # @see to_s
+    def inspect
+      digest
+    end
+
+    #
+    # Compares keys by digest
+    #
+    # @param [Key] other the key to compare with
+    #
+    # @return [true, false]
+    #
+    def ==(other)
+      digest == other.digest
+    end
+
+    #
+    # Returns all keys as an ordered array
+    #
+    # @return [Array] an ordered array with all keys
+    #
+    def to_a
+      [digest, queued, primed, locked, info, changelog, digests, expiring_digests]
+    end
+
+    private
+
+    def suffixed_key(variable)
+      "#{digest}:#{variable}"
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -5,10 +5,33 @@ module SidekiqUniqueJobs
     # Abstract base class for locks
     #
     # @abstract
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class BaseLock
+      extend Forwardable
+
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::Logging
 
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
+
+      #
+      # Validates that the sidekiq_options for the worker is valid
+      #
+      # @param [Hash] options the sidekiq_options given to the worker
+      #
+      # @return [void]
+      #
+      def self.validate_options(options = {})
+        Validator.validate(options)
+      end
+
+      # NOTE: Mainly used for a clean testing API
+      #
+      def_delegators :locksmith, :locked?
+
       # @param [Hash] item the Sidekiq job hash
       # @param [Proc] callback the callback to use after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
@@ -16,21 +39,22 @@ module SidekiqUniqueJobs
         @item       = item
         @callback   = callback
         @redis_pool = redis_pool
-        add_uniqueness_when_missing # Used to ease testing
+        @attempt    = 0
+        prepare_item # Used to ease testing
+        @lock_config = LockConfig.new(item)
       end
 
-      # Handles locking of sidekiq jobs.
-      #   Will call a conflict strategy if lock can't be achieved.
-      # @return [String] the sidekiq job id
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
       def lock
-        @attempt = 0
-        return item[JID_KEY] if locked?
-
-        if (token = locksmith.lock(item[LOCK_TIMEOUT_KEY]))
-          token
-        else
-          call_strategy
-        end
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
       end
 
       # Execute the job in the Sidekiq server processor
@@ -39,97 +63,102 @@ module SidekiqUniqueJobs
         raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
       end
 
-      # Unlocks the job from redis
-      # @return [String] sidekiq job id when successful
-      # @return [false] when unsuccessful
-      def unlock
-        locksmith.unlock(item[JID_KEY]) # Only signal to release the lock
-      end
-
-      # Deletes the job from redis if it is locked.
-      def delete
-        locksmith.delete # Soft delete (don't forcefully remove when expiration is set)
-      end
-
-      # Forcefully deletes the job from redis.
-      #   This is good for jobs when a previous lock was not unlocked
-      def delete!
-        locksmith.delete! # Force delete the lock
-      end
-
-      # Checks if the item has achieved a lock
-      # @return [true] when this jid has locked the job
-      # @return [false] when this jid has not locked the job
-      def locked?
-        locksmith.locked?(item[JID_KEY])
+      #
+      # The lock manager/client
+      #
+      # @api private
+      # @return [SidekiqUniqueJobs::Locksmith] the locksmith for this sidekiq job
+      #
+      def locksmith
+        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
       end
 
       private
 
-      def add_uniqueness_when_missing
-        return if item.key?(UNIQUE_DIGEST_KEY)
-
-        # The below should only be done to ease testing
-        # in production this will be done by the middleware
-        SidekiqUniqueJobs::Job.add_uniqueness(item)
-      end
-
-      def call_strategy
-        @attempt += 1
-        strategy.call { lock if replace? }
-      end
-
-      def replace?
-        strategy.replace? && attempt < 2
-      end
-
-      # The sidekiq job hash
-      # @return [Hash] the Sidekiq job hash
+      # @!attribute [r] item
+      #   @return [Hash<String, Object>] the Sidekiq job hash
       attr_reader :item
-
-      # The sidekiq redis pool
-      # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      # @!attribute [r] lock_config
+      #   @return [LockConfig] a lock configuration
+      attr_reader :lock_config
+      # @!attribute [r] redis_pool
+      #   @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
       attr_reader :redis_pool
-
-      # The sidekiq job hash
-      # @return [Proc] the callback to use after unlock
+      # @!attribute [r] callback
+      #   @return [Proc] the block to call after unlock
       attr_reader :callback
-
-      # The current attempt to lock the job
-      # @return [Integer] the numerical value of the attempt
+      # @!attribute [r] attempt
+      #   @return [Integer] the current locking attempt
       attr_reader :attempt
 
-      # The interface to the locking mechanism
-      # @return [SidekiqUniqueJobs::Locksmith]
-      def locksmith
-        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
+      #
+      # Eases testing by allowing the lock implementation to add the missing
+      # keys to the job hash.
+      #
+      #
+      # @return [void] the return value should be irrelevant
+      #
+      def prepare_item
+        return if item.key?(LOCK_DIGEST)
+
+        # The below should only be done to ease testing
+        # in production this will be done by the middleware
+        SidekiqUniqueJobs::Job.prepare(item)
       end
 
-      def with_cleanup
-        yield
-      rescue Sidekiq::Shutdown
-        log_info("Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!")
-        raise
-      else
-        unlock_with_callback
+      #
+      # Call whatever strategry that has been configured
+      #
+      # @param [Symbol] origin: the origin `:client` or `:server`
+      #
+      # @return [void] the return value is irrelevant
+      #
+      # @yieldparam [void] if a new job id was set and a block is given
+      # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
+      #   one specific situation to yield back to the middleware.
+      def call_strategy(origin:)
+        new_job_id = nil
+        strategy   = strategy_for(origin)
+        @attempt  += 1
+
+        strategy.call { new_job_id = lock if strategy.replace? && @attempt < 2 }
+        yield if new_job_id && block_given?
       end
 
-      def unlock_with_callback
-        return log_warn("might need to be unlocked manually") unless unlock
+      def unlock_and_callback
+        return callback_safely if locksmith.unlock
 
-        callback_safely
+        reflect(:unlock_failed, item)
       end
 
       def callback_safely
         callback&.call
-        item[JID_KEY]
+        item[JID]
       rescue StandardError
-        log_warn("unlocked successfully but the #after_unlock callback failed!")
+        reflect(:after_unlock_callback_failed, item)
         raise
       end
 
-      def strategy
-        @strategy ||= OnConflict.find_strategy(item[ON_CONFLICT_KEY]).new(item)
+      def strategy_for(origin)
+        case origin
+        when :client
+          client_strategy
+        when :server
+          server_strategy
+        else
+          raise SidekiqUniqueJobs::InvalidArgument,
+                "#origin needs to be either `:server` or `:client`"
+        end
+      end
+
+      def client_strategy
+        @client_strategy ||=
+          OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)
+      end
+
+      def server_strategy
+        @server_strategy ||=
+          OnConflict.find_strategy(lock_config.on_server_conflict).new(item, redis_pool)
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/client_validator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq client process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ClientValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:raise, :reject, :reschedule].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq client process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_client_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_client_conflict] = "#{on_conflict} is incompatible with the client process"
+        lock_config
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/server_validator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq server process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ServerValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:replace].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq server process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_server_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_server_conflict] = "#{on_conflict} is incompatible with the server process"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -11,36 +11,61 @@ module SidekiqUniqueJobs
     # See {#lock} for more information about the client.
     # See {#execute} for more information about the server
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilAndWhileExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(origin: :client, &block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: origin, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        if unlock
-          lock_on_failure do
-            runtime_lock.execute { return yield }
-          end
+        if locksmith.unlock
+          # ensure_relocked do
+          runtime_lock.execute { return yield }
+          # end
         else
-          log_warn "couldn't unlock digest: #{item[UNIQUE_DIGEST_KEY]} #{item[JID_KEY]}"
+          reflect(:unlock_failed, item)
         end
-      ensure
-        runtime_lock.delete!
-      end
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock(wait: 2)
 
-      def runtime_lock
-        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
+        raise
       end
 
       private
 
-      def lock_on_failure
+      def ensure_relocked
         yield
-        runtime_lock.delete!
       rescue Exception # rubocop:disable Lint/RescueException
-        log_error("Failed to execute job, restoring lock")
-        lock
+        reflect(:execution_failed, item)
+        locksmith.lock
+
         raise
       end
+
+      def runtime_lock
+        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -6,19 +6,41 @@ module SidekiqUniqueJobs
     # - Locks on perform_in or perform_async
     # - Unlocks after yielding to the worker's perform method
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuted < BaseLock
-      OK ||= "OK"
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        if locked?
-          with_cleanup { yield }
-        else
-          log_warn "the unique_key: #{item[UNIQUE_DIGEST_KEY]} is not locked, allowing job to silently complete"
-          nil
+        executed = locksmith.execute do
+          yield
+          unlock_and_callback
         end
+
+        reflect(:execution_failed, item) unless executed
+
+        nil
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -6,13 +6,37 @@ module SidekiqUniqueJobs
     # - Locks on perform_in or perform_async
     # - Unlocks before yielding to the worker's perform method
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        unlock_with_callback
+        callback_safely if locksmith.unlock
         yield
+      rescue StandardError => ex
+        reflect(:execution_failed, item, ex)
+        locksmith.lock(wait: 1)
+        raise
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -2,28 +2,40 @@
 
 module SidekiqUniqueJobs
   class Lock
-    # Locks jobs until the lock has expired
-    # - Locks on perform_in or perform_async
-    # - Unlocks when the expiration is hit
     #
-    # See {#lock} for more information about the client.
-    # See {#execute} for more information about the server
+    # UntilExpired locks until the job expires
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
-    class UntilExpired < BaseLock
-      # Prevents these locks from being unlocked
-      # @return [true] always returns true
-      def unlock
-        true
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class UntilExpired < UntilExecuted
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
       end
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
-      def execute
-        return unless locked?
+      def execute(&block)
+        executed = locksmith.execute(&block)
 
-        yield
-        # this lock does not handle after_unlock since we don't know when that would happen
+        reflect(:execution_failed, item) unless executed
       end
     end
   end