sidekiq-unique-jobs 4.0.18 → 7.1.28

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -1,12 +1,70 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  module Lock
-    class UntilAndWhileExecuting < UntilExecuting
-      def execute(callback)
-        lock = WhileExecuting.new(item, redis_pool)
-        lock.synchronize do
-          callback.call if unlock(:server)
-          yield
+  class Lock
+    # Locks jobs while the job is executing in the server process
+    # - Locks on perform_in or perform_async (see {UntilExecuting})
+    # - Unlocks before yielding to the worker's perform method (see {UntilExecuting})
+    # - Locks before yielding to the worker's perform method (see {WhileExecuting})
+    # - Unlocks after yielding to the worker's perform method (see {WhileExecuting})
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class UntilAndWhileExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(origin: :client, &block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: origin, &block)
+
+          return
         end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute
+        if locksmith.unlock
+          # ensure_relocked do
+          runtime_lock.execute { return yield }
+          # end
+        else
+          reflect(:unlock_failed, item)
+        end
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock(wait: 2)
+
+        raise
+      end
+
+      private
+
+      def ensure_relocked
+        yield
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock
+
+        raise
+      end
+
+      def runtime_lock
+        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -1,77 +1,47 @@
-module SidekiqUniqueJobs
-  module Lock
-    class UntilExecuted
-      OK ||= 'OK'.freeze
-
-      include SidekiqUniqueJobs::Unlockable
-
-      extend Forwardable
-      def_delegators :Sidekiq, :logger
-
-      def initialize(item, redis_pool = nil)
-        @item = item
-        @redis_pool = redis_pool
-      end
+# frozen_string_literal: true
 
-      def execute(callback, &blk)
-        operative = true
-        send(:after_yield_yield, &blk)
-      rescue Sidekiq::Shutdown
-        operative = false
-        raise
-      ensure
-        if operative && unlock(:server)
-          callback.call
-        else
-          logger.fatal { "the unique_key: #{unique_key} needs to be unlocked manually" }
+module SidekiqUniqueJobs
+  class Lock
+    # Locks jobs until the server is done executing the job
+    # - Locks on perform_in or perform_async
+    # - Unlocks after yielding to the worker's perform method
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class UntilExecuted < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
         end
-      end
 
-      def unlock(scope)
-        unless [:server, :api, :test].include?(scope)
-          raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
-        end
+        yield if block
 
-        unlock_by_key(unique_key, item[JID_KEY], redis_pool)
+        token
       end
 
-      # rubocop:disable MethodLength
-      def lock(scope)
-        if scope.to_sym != :client
-          raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute
+        executed = locksmith.execute do
+          yield
+          unlock_and_callback
         end
 
-        result = Scripts.call(:acquire_lock, redis_pool,
-                              keys: [unique_key],
-                              argv: [item[JID_KEY], max_lock_time])
-        case result
-        when 1
-          logger.debug { "successfully locked #{unique_key} for #{max_lock_time} seconds" }
-          true
-        when 0
-          logger.debug { "failed to acquire lock for #{unique_key}" }
-          false
-        else
-          raise "#{__method__} returned an unexpected value (#{result})"
-        end
-      end
-      # rubocop:enable MethodLength
-
-      def unique_key
-        @unique_key ||= UniqueArgs.digest(item)
-      end
+        reflect(:execution_failed, item) unless executed
 
-      def max_lock_time
-        @max_lock_time ||= QueueLockTimeoutCalculator.for_item(item).seconds
+        nil
       end
-
-      def after_yield_yield
-        yield
-      end
-
-      private
-
-      attr_reader :item, :redis_pool
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -1,9 +1,42 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  module Lock
-    class UntilExecuting < UntilExecuted
-      def execute(callback, &_block)
-        callback.call if unlock(:server)
+  class Lock
+    # Locks jobs until {#execute} starts
+    # - Locks on perform_in or perform_async
+    # - Unlocks before yielding to the worker's perform method
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class UntilExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute
+        callback_safely if locksmith.unlock
         yield
+      rescue StandardError => ex
+        reflect(:execution_failed, item, ex)
+        locksmith.lock(wait: 1)
+        raise
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # UntilExpired locks until the job expires
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class UntilExpired < UntilExecuted
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute(&block)
+        executed = locksmith.execute(&block)
+
+        reflect(:execution_failed, item) unless executed
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/validator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validator base class to avoid some duplication
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Validator
+      #
+      # @return [Hash] a hash mapping of deprecated keys and their new value
+      DEPRECATED_KEYS = {
+        UNIQUE.to_sym => LOCK.to_sym,
+        UNIQUE_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
+        LOCK_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
+        UNIQUE_PREFIX.to_sym => LOCK_PREFIX.to_sym,
+      }.freeze
+
+      #
+      # Shorthand for `new(options).validate`
+      #
+      # @param [Hash] options the sidekiq_options for the worker being validated
+      #
+      # @return [LockConfig] the lock configuration with errors if any
+      #
+      def self.validate(options)
+        new(options).validate
+      end
+
+      #
+      # @!attribute [r] lock_config
+      #   @return [LockConfig] the lock configuration for this worker
+      attr_reader :lock_config
+
+      #
+      # Initialize a new validator
+      #
+      # @param [Hash] options the sidekiq_options for the worker being validated
+      #
+      def initialize(options)
+        @options     = options.transform_keys(&:to_sym)
+        @lock_config = LockConfig.new(options)
+        handle_deprecations
+      end
+
+      #
+      # Validate the workers lock configuration
+      #
+      #
+      # @return [LockConfig] the lock configuration with errors if any
+      #
+      def validate
+        case lock_config.type
+        when :while_executing
+          validate_server
+        when :until_executing
+          validate_client
+        else
+          validate_client
+          validate_server
+        end
+
+        lock_config
+      end
+
+      #
+      # Validate deprecated keys
+      #   adds useful information about how to proceed with fixing handle_deprecations
+      #
+      # @return [void]
+      #
+      def handle_deprecations
+        DEPRECATED_KEYS.each do |old, new|
+          next unless @options.key?(old)
+
+          lock_config.errors[old] = "is deprecated, use `#{new}: #{@options[old]}` instead."
+        end
+      end
+
+      #
+      # Validates the client configuration
+      #
+      def validate_client
+        ClientValidator.validate(lock_config)
+      end
+
+      #
+      # Validates the server configuration
+      #
+      def validate_server
+        ServerValidator.validate(lock_config)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -1,49 +1,67 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  module Lock
-    class WhileExecuting
-      def self.synchronize(item, redis_pool = nil)
-        new(item, redis_pool).synchronize { yield }
-      end
+  class Lock
+    # Locks jobs while the job is executing in the server process
+    # - Locks before yielding to the worker's perform method
+    # - Unlocks after yielding to the worker's perform method
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class WhileExecuting < BaseLock
+      RUN_SUFFIX = ":RUN"
 
-      def initialize(item, redis_pool = nil)
-        @item = item
-        @mutex = Mutex.new
-        @redis_pool = redis_pool
-        @unique_digest = "#{create_digest}:run"
-      end
+      include SidekiqUniqueJobs::OptionsWithFallback
+      include SidekiqUniqueJobs::Logging::Middleware
 
-      def synchronize
-        @mutex.lock
-        sleep 0.001 until locked?
-
-        yield
-      rescue Sidekiq::Shutdown
-        logger.fatal { "the unique_key: #{@unique_digest} needs to be unlocked manually" }
-        raise
-      ensure
-        SidekiqUniqueJobs.connection(@redis_pool) { |c| c.del @unique_digest }
-        @mutex.unlock
+      # @param [Hash] item the Sidekiq job hash
+      # @param [Proc] callback callback to call after unlock
+      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+      #
+      def initialize(item, callback, redis_pool = nil)
+        super(item, callback, redis_pool)
+        append_unique_key_suffix
       end
 
-      def locked?
-        Scripts.call(:synchronize, @redis_pool,
-                     keys: [@unique_digest],
-                     argv: [Time.now.to_i, max_lock_time]) == 1
-      end
+      # Simulate that a client lock was achieved.
+      #   These locks should only ever be created in the server process.
+      # @return [true] always returns true
+      def lock
+        job_id = item[JID]
+        yield if block_given?
 
-      def max_lock_time
-        @max_lock_time ||= RunLockTimeoutCalculator.for_item(@item).seconds
+        job_id
       end
 
-      def execute(_callback)
-        synchronize do
-          yield
+      # Executes in the Sidekiq server process.
+      #   These jobs are locked in the server process not from the client
+      # @yield to the worker class perform method
+      def execute(&block)
+        with_logging_context do
+          executed = locksmith.execute do
+            yield
+            callback_safely if locksmith.unlock
+          ensure
+            locksmith.unlock
+          end
+
+          unless executed
+            reflect(:execution_failed, item)
+            call_strategy(origin: :server, &block)
+          end
         end
       end
 
-      def create_digest
-        @unique_digest ||= @item[UNIQUE_DIGEST_KEY]
-        @unique_digest ||= SidekiqUniqueJobs::UniqueArgs.digest(@item)
+      private
+
+      # This is safe as the base_lock always creates a new digest
+      #   The append there for needs to be done every time
+      def append_unique_key_suffix
+        return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
+
+        item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Locks jobs while executing
+    #   Locks from the server process
+    #   Unlocks after the server is done processing
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class WhileExecutingReject < WhileExecuting
+      # Overridden with a forced {OnConflict::Reject} strategy
+      # @return [OnConflict::Reject] a reject strategy
+      def server_strategy
+        @server_strategy ||= OnConflict.find_strategy(:reject).new(item, redis_pool)
+      end
+    end
+  end
+end