sidekiq-unique-jobs 7.0.2 → 7.1.12

data/lib/sidekiq_unique_jobs/changelog.rb

@@ -68,8 +68,8 @@ module SidekiqUniqueJobs
         end
 
         [
-          total_size,
-          result[0], # next_cursor
+          total_size.to_i,
+          result[0].to_i, # next_cursor
           result[1].map { |entry| load_json(entry[0]) }, # entries
         ]
       end

data/lib/sidekiq_unique_jobs/config.rb

@@ -16,6 +16,8 @@ module SidekiqUniqueJobs
                                                    :reaper_count,
                                                    :reaper_interval,
                                                    :reaper_timeout,
+                                                   :reaper_resurrector_interval,
+                                                   :reaper_resurrector_enabled,
                                                    :lock_info,
                                                    :raise_on_config_error,
                                                    :current_redis_version)
@@ -109,6 +111,14 @@ module SidekiqUniqueJobs
     #
     # @return [10] stop reaper after 10 seconds
     REAPER_TIMEOUT        = 10
+    #
+    # @return [3600] check if reaper is dead each 3600 seconds
+    REAPER_RESURRECTOR_INTERVAL = 3600
+
+    #
+    # @return [false] enable reaper resurrector
+    REAPER_RESURRECTOR_ENABLED = false
+
     #
     # @return [false] while useful it also adds overhead so disable lock_info by default
     USE_LOCK_INFO         = false
@@ -178,32 +188,73 @@ module SidekiqUniqueJobs
         REAPER_COUNT,
         REAPER_INTERVAL,
         REAPER_TIMEOUT,
+        REAPER_RESURRECTOR_INTERVAL,
+        REAPER_RESURRECTOR_ENABLED,
         USE_LOCK_INFO,
         RAISE_ON_CONFIG_ERROR,
         REDIS_VERSION,
       )
     end
 
+    #
+    # Set the default_lock_ttl
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [<type>] <description>
+    #
     def default_lock_ttl=(obj)
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl=` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl=` instead."
       self.lock_ttl = obj
     end
 
+    #
+    # Set new value for default_lock_timeout
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [Integer]
+    #
     def default_lock_timeout=(obj)
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout=` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout=` instead."
       self.lock_timeout = obj
     end
 
+    #
+    # Default lock TTL (Time To Live)
+    # @deprecated
+    #
+    # @return [nil, Integer] configured value or nil
+    #
     def default_lock_ttl
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl` instead."
       lock_ttl
     end
 
+    #
+    # Default Lock Timeout
+    # @deprecated
+    #
+    #
+    # @return [nil, Integer] configured value or nil
+    #
     def default_lock_timeout
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout` instead."
       lock_timeout
     end
 
+    #
+    # Memoized variable to get the class name
+    #
+    #
+    # @return [String] name of the class
+    #
     def class_name
       @class_name ||= self.class.name
     end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -6,49 +6,48 @@
 # @author Mikael Henriksson <mikael@mhenrixon.com>
 #
 module SidekiqUniqueJobs
-  ARGS                  ||= "args"
-  APARTMENT             ||= "apartment"
-  AT                    ||= "at"
-  CHANGELOGS            ||= "uniquejobs:changelog"
-  CLASS                 ||= "class"
-  CREATED_AT            ||= "created_at"
-  DEAD_VERSION          ||= "uniquejobs:dead"
-  DIGESTS               ||= "uniquejobs:digests"
-  ERRORS                ||= "errors"
-  JID                   ||= "jid"
-  LIMIT                 ||= "limit"
-  LIVE_VERSION          ||= "uniquejobs:live"
-  LOCK                  ||= "lock"
-  LOCK_ARGS             ||= "lock_args"
-  LOCK_ARGS_METHOD      ||= "lock_args_method"
-  LOCK_DIGEST           ||= "lock_digest"
-  LOCK_EXPIRATION       ||= "lock_expiration"
-  LOCK_INFO             ||= "lock_info"
-  LOCK_LIMIT            ||= "lock_limit"
-  LOCK_PREFIX           ||= "lock_prefix"
-  LOCK_TIMEOUT          ||= "lock_timeout"
-  LOCK_TTL              ||= "lock_ttl"
-  LOCK_TYPE             ||= "lock_type"
-  LOG_DUPLICATE         ||= "log_duplicate"
-  ON_CLIENT_CONFLICT    ||= "on_client_conflict"
-  ON_CONFLICT           ||= "on_conflict"
-  ON_SERVER_CONFLICT    ||= "on_server_conflict"
-  PAYLOAD               ||= "payload"
-  PROCESSES             ||= "processes"
-  QUEUE                 ||= "queue"
-  RETRY                 ||= "retry"
-  SCHEDULE              ||= "schedule"
-  TIME                  ||= "time"
-  TIMEOUT               ||= "timeout"
-  TTL                   ||= "ttl"
-  TYPE                  ||= "type"
-  UNIQUE                ||= "unique"
-  UNIQUE_ACROSS_QUEUES  ||= "unique_across_queues"
-  UNIQUE_ACROSS_WORKERS ||= "unique_across_workers"
-  UNIQUE_ARGS           ||= "unique_args"
-  UNIQUE_ARGS_METHOD    ||= "unique_args_method"
-  UNIQUE_DIGEST         ||= "unique_digest"
-  UNIQUE_PREFIX         ||= "unique_prefix"
-  UNIQUE_REAPER         ||= "uniquejobs:reaper"
-  WORKER                ||= "worker"
+  ARGS                  = "args"
+  APARTMENT             = "apartment"
+  AT                    = "at"
+  CHANGELOGS            = "uniquejobs:changelog"
+  CLASS                 = "class"
+  CREATED_AT            = "created_at"
+  DEAD_VERSION          = "uniquejobs:dead"
+  DIGESTS               = "uniquejobs:digests"
+  ERRORS                = "errors"
+  JID                   = "jid"
+  LIMIT                 = "limit"
+  LIVE_VERSION          = "uniquejobs:live"
+  LOCK                  = "lock"
+  LOCK_ARGS             = "lock_args"
+  LOCK_ARGS_METHOD      = "lock_args_method"
+  LOCK_DIGEST           = "lock_digest"
+  LOCK_EXPIRATION       = "lock_expiration"
+  LOCK_INFO             = "lock_info"
+  LOCK_LIMIT            = "lock_limit"
+  LOCK_PREFIX           = "lock_prefix"
+  LOCK_TIMEOUT          = "lock_timeout"
+  LOCK_TTL              = "lock_ttl"
+  LOCK_TYPE             = "lock_type"
+  ON_CLIENT_CONFLICT    = "on_client_conflict"
+  ON_CONFLICT           = "on_conflict"
+  ON_SERVER_CONFLICT    = "on_server_conflict"
+  PAYLOAD               = "payload"
+  PROCESSES             = "processes"
+  QUEUE                 = "queue"
+  RETRY                 = "retry"
+  SCHEDULE              = "schedule"
+  TIME                  = "time"
+  TIMEOUT               = "timeout"
+  TTL                   = "ttl"
+  TYPE                  = "type"
+  UNIQUE                = "unique"
+  UNIQUE_ACROSS_QUEUES  = "unique_across_queues"
+  UNIQUE_ACROSS_WORKERS = "unique_across_workers"
+  UNIQUE_ARGS           = "unique_args"
+  UNIQUE_ARGS_METHOD    = "unique_args_method"
+  UNIQUE_DIGEST         = "unique_digest"
+  UNIQUE_PREFIX         = "unique_prefix"
+  UNIQUE_REAPER         = "uniquejobs:reaper"
+  WORKER                = "worker"
 end

data/lib/sidekiq_unique_jobs/deprecation.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Deprecation provides logging of deprecations
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Deprecation
+    #
+    # Mute warnings from this gem in a threaded context
+    #
+    #
+    # @return [void] <description>
+    #
+    # @yieldreturn [void]
+    def self.muted
+      orig_val = Thread.current[:uniquejobs_mute_deprecations]
+      Thread.current[:uniquejobs_mute_deprecations] = true
+      yield
+    ensure
+      Thread.current[:uniquejobs_mute_deprecations] = orig_val
+    end
+
+    #
+    # Check if deprecation warnings have been muted
+    #
+    #
+    # @return [true,false]
+    #
+    def self.muted?
+      Thread.current[:uniquejobs_mute_deprecations] == true
+    end
+
+    #
+    # Warn about deprecation
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      warn "DEPRECATION WARNING: #{msg}"
+      nil
+    end
+
+    #
+    # Warn about deprecation and provide a context
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn_with_backtrace(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      trace = "\n\nCALLED FROM:\n#{caller.join("\n")}"
+      warn(msg + trace)
+
+      nil
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/digests.rb

@@ -31,7 +31,8 @@ module SidekiqUniqueJobs
     #
     # @param [String] pattern a key pattern to match with
     # @param [Integer] count the maximum number
-    # @return [Array<String>] with unique digests
+    # @return [Hash<String,Float>] Hash mapping of digest matching the given pattern and score
+
     def delete_by_pattern(pattern, count: DEFAULT_COUNT)
       result, elapsed = timed do
         digests = entries(pattern: pattern, count: count).keys
@@ -80,11 +81,7 @@ module SidekiqUniqueJobs
       options[:match] = pattern
       options[:count] = count
 
-      result = redis { |conn| conn.zscan_each(key, **options).to_a }
-
-      result.each_with_object({}) do |entry, hash|
-        hash[entry[0]] = entry[1]
-      end
+      redis { |conn| conn.zscan_each(key, **options).to_a }.to_h
     end
 
     #
@@ -104,8 +101,8 @@ module SidekiqUniqueJobs
         end
 
         [
-          total_size,
-          digests[0], # next_cursor
+          total_size.to_i,
+          digests[0].to_i, # next_cursor
           digests[1].map { |digest, score| Lock.new(digest, time: score) }, # entries
         ]
       end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -18,6 +18,16 @@ module SidekiqUniqueJobs
     end
   end
 
+  #
+  # Raised when no block was given
+  #
+  class NoBlockGiven < SidekiqUniqueJobs::UniqueJobsError; end
+
+  #
+  # Raised when a notification has been mistyped
+  #
+  class NoSuchNotificationError < UniqueJobsError; end
+
   #
   # Error raised when trying to add a duplicate lock
   #

data/lib/sidekiq_unique_jobs/json.rb

@@ -20,6 +20,13 @@ module SidekiqUniqueJobs
       ::JSON.parse(string)
     end
 
+    #
+    # Prevents trying JSON.load from raising errors given argument is a hash
+    #
+    # @param [String, Hash] string the JSON string to parse
+    #
+    # @return [Hash,Array]
+    #
     def safe_load_json(string)
       return string if string.is_a?(Hash)
 

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -7,8 +7,16 @@ module SidekiqUniqueJobs
     # @abstract
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class BaseLock
+      extend Forwardable
+
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::Logging
 
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
+
       #
       # Validates that the sidekiq_options for the worker is valid
       #
@@ -20,6 +28,10 @@ module SidekiqUniqueJobs
         Validator.validate(options)
       end
 
+      # NOTE: Mainly used for a clean testing API
+      #
+      def_delegators :locksmith, :locked?
+
       # @param [Hash] item the Sidekiq job hash
       # @param [Proc] callback the callback to use after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
@@ -41,10 +53,8 @@ module SidekiqUniqueJobs
       #
       # @yield to the caller when given a block
       #
-      def lock(&block)
-        return call_strategy unless (locked_token = locksmith.lock(&block))
-
-        locked_token
+      def lock
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
       end
 
       # Execute the job in the Sidekiq server processor
@@ -53,31 +63,6 @@ module SidekiqUniqueJobs
         raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
       end
 
-      # Unlocks the job from redis
-      # @return [String] sidekiq job id when successful
-      # @return [false] when unsuccessful
-      def unlock
-        locksmith.unlock # Only signal to release the lock
-      end
-
-      # Deletes the job from redis if it is locked.
-      def delete
-        locksmith.delete # Soft delete (don't forcefully remove when expiration is set)
-      end
-
-      # Forcefully deletes the job from redis.
-      #   This is good for jobs when a previous lock was not unlocked
-      def delete!
-        locksmith.delete! # Force delete the lock
-      end
-
-      # Checks if the item has achieved a lock
-      # @return [true] when this jid has locked the job
-      # @return [false] when this jid has not locked the job
-      def locked?
-        locksmith.locked?
-      end
-
       #
       # The lock manager/client
       #
@@ -90,23 +75,6 @@ module SidekiqUniqueJobs
 
       private
 
-      def prepare_item
-        return if item.key?(LOCK_DIGEST)
-
-        # The below should only be done to ease testing
-        # in production this will be done by the middleware
-        SidekiqUniqueJobs::Job.prepare(item)
-      end
-
-      def call_strategy
-        @attempt += 1
-        client_strategy.call { lock if replace? }
-      end
-
-      def replace?
-        client_strategy.replace? && attempt < 2
-      end
-
       # @!attribute [r] item
       #   @return [Hash<String, Object>] the Sidekiq job hash
       attr_reader :item
@@ -123,21 +91,66 @@ module SidekiqUniqueJobs
       #   @return [Integer] the current locking attempt
       attr_reader :attempt
 
-      def unlock_with_callback
-        return log_warn("might need to be unlocked manually") unless unlock
+      #
+      # Eases testing by allowing the lock implementation to add the missing
+      # keys to the job hash.
+      #
+      #
+      # @return [void] the return value should be irrelevant
+      #
+      def prepare_item
+        return if item.key?(LOCK_DIGEST)
 
-        callback_safely
-        item[JID]
+        # The below should only be done to ease testing
+        # in production this will be done by the middleware
+        SidekiqUniqueJobs::Job.prepare(item)
+      end
+
+      #
+      # Call whatever strategry that has been configured
+      #
+      # @param [Symbol] origin: the origin `:client` or `:server`
+      #
+      # @return [void] the return value is irrelevant
+      #
+      # @yieldparam [void] if a new job id was set and a block is given
+      # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
+      #   one specific situation to yield back to the middleware.
+      def call_strategy(origin:)
+        new_job_id = nil
+        strategy   = strategy_for(origin)
+        @attempt  += 1
+
+        strategy.call { new_job_id = lock if strategy.replace? && @attempt < 2 }
+        yield if new_job_id && block_given?
+      end
+
+      def unlock_and_callback
+        return callback_safely if locksmith.unlock
+
+        reflect(:unlock_failed, item)
       end
 
       def callback_safely
         callback&.call
         item[JID]
       rescue StandardError
-        log_warn("unlocked successfully but the #after_unlock callback failed!")
+        reflect(:after_unlock_callback_failed, item)
         raise
       end
 
+      def strategy_for(origin)
+        case origin
+        when :client
+          client_strategy
+        when :server
+          server_strategy
+        else
+          raise SidekiqUniqueJobs::InvalidArgument,
+                "#origin needs to be either `:server` or `:client`"
+        end
+      end
+
       def client_strategy
         @client_strategy ||=
           OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -13,30 +13,58 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilAndWhileExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(origin: :client, &block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: origin, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        if unlock
-          lock_on_failure do
-            runtime_lock.execute { return yield }
-          end
+        if locksmith.unlock
+          # ensure_relocked do
+          runtime_lock.execute { return yield }
+          # end
         else
-          log_warn "couldn't unlock digest: #{item[LOCK_DIGEST]} #{item[JID]}"
+          reflect(:unlock_failed, item)
         end
+      rescue Exception # rubocop:disable Lint/RescueException
+        reflect(:execution_failed, item)
+        locksmith.lock(wait: 2)
+
+        raise
       end
 
       private
 
-      def lock_on_failure
+      def ensure_relocked
         yield
       rescue Exception # rubocop:disable Lint/RescueException
-        log_error("Runtime lock failed to execute job, restoring server lock")
-        lock
+        reflect(:execution_failed, item)
+        locksmith.lock
+
         raise
       end
 
       def runtime_lock
-        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
+        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -8,16 +8,34 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuted < BaseLock
-      OK ||= "OK"
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        lock do
+        locksmith.execute do
           yield
-          unlock_with_callback
-          callback_safely
-          item[JID]
+          unlock_and_callback
         end
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -8,10 +8,30 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        unlock_with_callback
+        callback_safely if locksmith.unlock
         yield
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -8,6 +8,33 @@ module SidekiqUniqueJobs
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     class UntilExpired < UntilExecuted
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
+
+        token
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute(&block)
+        locksmith.execute(&block)
+      end
     end
   end
 end