sidekiq-unique-jobs 3.0.11 → 8.0.10

data/lib/sidekiq_unique_jobs/key.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Key class wraps logic dealing with various lock keys
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Key
+    #
+    # @!attribute [r] digest
+    #   @return [String] the digest key for which keys are created
+    attr_reader :digest
+    #
+    # @!attribute [r] queued
+    #   @return [String] the list key with queued job_id's
+    attr_reader :queued
+    #
+    # @!attribute [r] primed
+    #   @return [String] the list key with primed job_id's
+    attr_reader :primed
+    #
+    # @!attribute [r] locked
+    #   @return [String] the hash key with locked job_id's
+    attr_reader :locked
+    #
+    # @!attribute [r] info
+    #   @return [String] information about the lock
+    attr_reader :info
+    #
+    # @!attribute [r] changelog
+    #   @return [String] the zset with changelog entries
+    attr_reader :changelog
+    #
+    # @!attribute [r] digests
+    #   @return [String] the zset with locked digests
+    attr_reader :digests
+    #
+    # @!attribute [r] expiring_digests
+    #   @return [String] the zset with locked expiring_digests
+    attr_reader :expiring_digests
+
+    #
+    # Initialize a new Key
+    #
+    # @param [String] digest the digest to use as key
+    #
+    def initialize(digest)
+      @digest           = digest
+      @queued           = suffixed_key("QUEUED")
+      @primed           = suffixed_key("PRIMED")
+      @locked           = suffixed_key("LOCKED")
+      @info             = suffixed_key("INFO")
+      @changelog        = CHANGELOGS
+      @digests          = DIGESTS
+      @expiring_digests = EXPIRING_DIGESTS
+    end
+
+    #
+    # Provides the only important information about this keys
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      digest
+    end
+
+    # @see to_s
+    def inspect
+      digest
+    end
+
+    #
+    # Compares keys by digest
+    #
+    # @param [Key] other the key to compare with
+    #
+    # @return [true, false]
+    #
+    def ==(other)
+      digest == other.digest
+    end
+
+    #
+    # Returns all keys as an ordered array
+    #
+    # @return [Array] an ordered array with all keys
+    #
+    def to_a
+      [digest, queued, primed, locked, info, changelog, digests, expiring_digests]
+    end
+
+    private
+
+    def suffixed_key(variable)
+      "#{digest}:#{variable}"
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Abstract base class for locks
+    #
+    # @abstract
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class BaseLock
+      extend Forwardable
+
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::Logging
+
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
+
+      #
+      # Validates that the sidekiq_options for the worker is valid
+      #
+      # @param [Hash] options the sidekiq_options given to the worker
+      #
+      # @return [void]
+      #
+      def self.validate_options(options = {})
+        Validator.validate(options)
+      end
+
+      # NOTE: Mainly used for a clean testing API
+      #
+      def_delegators :locksmith, :locked?
+
+      # @param [Hash] item the Sidekiq job hash
+      # @param [Proc] callback the callback to use after unlock
+      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+      def initialize(item, callback, redis_pool = nil)
+        @item       = item
+        @callback   = callback
+        @redis_pool = redis_pool
+        @attempt    = 0
+        prepare_item # Used to ease testing
+        @lock_config = LockConfig.new(item)
+      end
+
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
+      end
+
+      # Execute the job in the Sidekiq server processor
+      # @raise [NotImplementedError] needs to be implemented in child class
+      def execute
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
+      end
+
+      #
+      # The lock manager/client
+      #
+      # @api private
+      # @return [SidekiqUniqueJobs::Locksmith] the locksmith for this sidekiq job
+      #
+      def locksmith
+        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
+      end
+
+      private
+
+      # @!attribute [r] item
+      #   @return [Hash<String, Object>] the Sidekiq job hash
+      attr_reader :item
+      # @!attribute [r] lock_config
+      #   @return [LockConfig] a lock configuration
+      attr_reader :lock_config
+      # @!attribute [r] redis_pool
+      #   @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      attr_reader :redis_pool
+      # @!attribute [r] callback
+      #   @return [Proc] the block to call after unlock
+      attr_reader :callback
+      # @!attribute [r] attempt
+      #   @return [Integer] the current locking attempt
+      attr_reader :attempt
+
+      #
+      # Eases testing by allowing the lock implementation to add the missing
+      # keys to the job hash.
+      #
+      #
+      # @return [void] the return value should be irrelevant
+      #
+      def prepare_item
+        return if item.key?(LOCK_DIGEST)
+
+        # The below should only be done to ease testing
+        # in production this will be done by the middleware
+        SidekiqUniqueJobs::Job.prepare(item)
+      end
+
+      #
+      # Call whatever strategry that has been configured
+      #
+      # @param [Symbol] origin the origin `:client` or `:server`
+      #
+      # @return [void] the return value is irrelevant
+      #
+      # @yieldparam [void] if a new job id was set and a block is given
+      # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
+      #   one specific situation to yield back to the middleware.
+      def call_strategy(origin:)
+        new_job_id = nil
+        strategy   = strategy_for(origin)
+        @attempt  += 1
+
+        strategy.call { new_job_id = lock if strategy.replace? && @attempt < 2 }
+        yield if new_job_id && block_given?
+      end
+
+      def unlock_and_callback
+        return callback_safely if locksmith.unlock
+
+        reflect(:unlock_failed, item)
+      end
+
+      def callback_safely
+        callback&.call
+        item[JID]
+      rescue StandardError
+        reflect(:after_unlock_callback_failed, item)
+        raise
+      end
+
+      def strategy_for(origin)
+        case origin
+        when :client
+          client_strategy
+        when :server
+          server_strategy
+        else
+          raise SidekiqUniqueJobs::InvalidArgument,
+                "#origin needs to be either `:server` or `:client`"
+        end
+      end
+
+      def client_strategy
+        @client_strategy ||=
+          OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)
+      end
+
+      def server_strategy
+        @server_strategy ||=
+          OnConflict.find_strategy(lock_config.on_server_conflict).new(item, redis_pool)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/client_validator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq client process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ClientValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:raise, :reject, :reschedule].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq client process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_client_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_client_conflict] = "#{on_conflict} is incompatible with the client process"
+        lock_config
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/server_validator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq server process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ServerValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:replace].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq server process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_server_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_server_conflict] = "#{on_conflict} is incompatible with the server process"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Locks jobs while the job is executing in the server process
+    # - Locks on perform_in or perform_async (see {UntilExecuting})
+    # - Unlocks before yielding to the worker's perform method (see {UntilExecuting})
+    # - Locks before yielding to the worker's perform method (see {WhileExecuting})
+    # - Unlocks after yielding to the worker's perform method (see {WhileExecuting})
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class UntilAndWhileExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(origin: :client, &block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: origin, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute
+        if locksmith.unlock
+          # ensure_relocked do
+          runtime_lock.execute { return yield }
+          # end
+        else
+          reflect(:unlock_failed, item)
+        end
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock(wait: 2)
+
+        raise
+      end
+
+      private
+
+      def ensure_relocked
+        yield
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock
+
+        raise
+      end
+
+      def runtime_lock
+        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Locks jobs until the server is done executing the job
+    # - Locks on perform_in or perform_async
+    # - Unlocks after yielding to the worker's perform method
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class UntilExecuted < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute
+        executed = locksmith.execute do
+          yield
+        ensure
+          unlock_and_callback
+        end
+
+        reflect(:execution_failed, item) unless executed
+
+        nil
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Locks jobs until {#execute} starts
+    # - Locks on perform_in or perform_async
+    # - Unlocks before yielding to the worker's perform method
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class UntilExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute
+        callback_safely if locksmith.unlock
+        yield
+      rescue StandardError => ex
+        reflect(:execution_failed, item, ex)
+        locksmith.lock(wait: 1)
+        raise
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # UntilExpired locks until the job expires
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class UntilExpired < UntilExecuted
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute(&block)
+        executed = locksmith.execute(&block)
+
+        reflect(:execution_failed, item) unless executed
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/validator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validator base class to avoid some duplication
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Validator
+      #
+      # @return [Hash] a hash mapping of deprecated keys and their new value
+      DEPRECATED_KEYS = {
+        UNIQUE.to_sym => LOCK.to_sym,
+        UNIQUE_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
+        LOCK_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
+        UNIQUE_PREFIX.to_sym => LOCK_PREFIX.to_sym,
+      }.freeze
+
+      #
+      # Shorthand for `new(options).validate`
+      #
+      # @param [Hash] options the sidekiq_options for the worker being validated
+      #
+      # @return [LockConfig] the lock configuration with errors if any
+      #
+      def self.validate(options)
+        new(options).validate
+      end
+
+      #
+      # @!attribute [r] lock_config
+      #   @return [LockConfig] the lock configuration for this worker
+      attr_reader :lock_config
+
+      #
+      # Initialize a new validator
+      #
+      # @param [Hash] options the sidekiq_options for the worker being validated
+      #
+      def initialize(options)
+        @options     = options.transform_keys(&:to_sym)
+        @lock_config = LockConfig.new(options)
+        handle_deprecations
+      end
+
+      #
+      # Validate the workers lock configuration
+      #
+      #
+      # @return [LockConfig] the lock configuration with errors if any
+      #
+      def validate
+        case lock_config.type
+        when :while_executing
+          validate_server
+        when :until_executing
+          validate_client
+        else
+          validate_client
+          validate_server
+        end
+
+        lock_config
+      end
+
+      #
+      # Validate deprecated keys
+      #   adds useful information about how to proceed with fixing handle_deprecations
+      #
+      # @return [void]
+      #
+      def handle_deprecations
+        DEPRECATED_KEYS.each do |old, new|
+          next unless @options.key?(old)
+
+          lock_config.errors[old] = "is deprecated, use `#{new}: #{@options[old]}` instead."
+        end
+      end
+
+      #
+      # Validates the client configuration
+      #
+      def validate_client
+        ClientValidator.validate(lock_config)
+      end
+
+      #
+      # Validates the server configuration
+      #
+      def validate_server
+        ServerValidator.validate(lock_config)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Locks jobs while the job is executing in the server process
+    # - Locks before yielding to the worker's perform method
+    # - Unlocks after yielding to the worker's perform method
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class WhileExecuting < BaseLock
+      #
+      # @return [String] returns :RUN
+      RUN_SUFFIX = ":RUN"
+
+      include SidekiqUniqueJobs::OptionsWithFallback
+      include SidekiqUniqueJobs::Logging::Middleware
+
+      # @param [Hash] item the Sidekiq job hash
+      # @param [Proc] callback callback to call after unlock
+      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+      #
+      def initialize(item, callback, redis_pool = nil)
+        super(item, callback, redis_pool)
+        append_unique_key_suffix
+      end
+
+      # Simulate that a client lock was achieved.
+      #   These locks should only ever be created in the server process.
+      # @return [true] always returns true
+      def lock
+        job_id = item[JID]
+        yield if block_given?
+
+        job_id
+      end
+
+      # Executes in the Sidekiq server process.
+      #   These jobs are locked in the server process not from the client
+      # @yield to the worker class perform method
+      def execute(&block)
+        with_logging_context do
+          executed = locksmith.execute do
+            yield
+            item[JID]
+          ensure
+            unlock_and_callback
+          end
+
+          unless executed
+            reflect(:execution_failed, item)
+            call_strategy(origin: :server, &block)
+          end
+        end
+      end
+
+      private
+
+      # This is safe as the base_lock always creates a new digest
+      #   The append there for needs to be done every time
+      def append_unique_key_suffix
+        return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
+
+        item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
+      end
+    end
+  end
+end