sidekiq-unique-jobs 7.1.6 → 7.1.30

data/lib/sidekiq_unique_jobs/config.rb

@@ -8,6 +8,7 @@ module SidekiqUniqueJobs
                                                    :enabled,
                                                    :lock_prefix,
                                                    :logger,
+                                                   :logger_enabled,
                                                    :locks,
                                                    :strategies,
                                                    :debug_lua,
@@ -91,6 +92,9 @@ module SidekiqUniqueJobs
     # @return [nil]
     LOCK_TTL              = nil
     #
+    # @return [true,false] by default false (don't disable logger)
+    LOGGER_ENABLED        = true
+    #
     # @return [true] by default the gem is enabled
     ENABLED               = true
     #
@@ -180,6 +184,7 @@ module SidekiqUniqueJobs
         ENABLED,
         PREFIX,
         Sidekiq.logger,
+        LOGGER_ENABLED,
         LOCKS,
         STRATEGIES,
         DEBUG_LUA,
@@ -196,26 +201,65 @@ module SidekiqUniqueJobs
       )
     end
 
+    #
+    # Set the default_lock_ttl
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [<type>] <description>
+    #
     def default_lock_ttl=(obj)
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl=` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl=` instead."
       self.lock_ttl = obj
     end
 
+    #
+    # Set new value for default_lock_timeout
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [Integer]
+    #
     def default_lock_timeout=(obj)
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout=` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout=` instead."
       self.lock_timeout = obj
     end
 
+    #
+    # Default lock TTL (Time To Live)
+    # @deprecated
+    #
+    # @return [nil, Integer] configured value or nil
+    #
     def default_lock_ttl
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl` instead."
       lock_ttl
     end
 
+    #
+    # Default Lock Timeout
+    # @deprecated
+    #
+    #
+    # @return [nil, Integer] configured value or nil
+    #
     def default_lock_timeout
-      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout` instead."
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout` instead."
       lock_timeout
     end
 
+    #
+    # Memoized variable to get the class name
+    #
+    #
+    # @return [String] name of the class
+    #
     def class_name
       @class_name ||= self.class.name
     end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -6,48 +6,49 @@
 # @author Mikael Henriksson <mikael@mhenrixon.com>
 #
 module SidekiqUniqueJobs
-  ARGS                  ||= "args"
-  APARTMENT             ||= "apartment"
-  AT                    ||= "at"
-  CHANGELOGS            ||= "uniquejobs:changelog"
-  CLASS                 ||= "class"
-  CREATED_AT            ||= "created_at"
-  DEAD_VERSION          ||= "uniquejobs:dead"
-  DIGESTS               ||= "uniquejobs:digests"
-  ERRORS                ||= "errors"
-  JID                   ||= "jid"
-  LIMIT                 ||= "limit"
-  LIVE_VERSION          ||= "uniquejobs:live"
-  LOCK                  ||= "lock"
-  LOCK_ARGS             ||= "lock_args"
-  LOCK_ARGS_METHOD      ||= "lock_args_method"
-  LOCK_DIGEST           ||= "lock_digest"
-  LOCK_EXPIRATION       ||= "lock_expiration"
-  LOCK_INFO             ||= "lock_info"
-  LOCK_LIMIT            ||= "lock_limit"
-  LOCK_PREFIX           ||= "lock_prefix"
-  LOCK_TIMEOUT          ||= "lock_timeout"
-  LOCK_TTL              ||= "lock_ttl"
-  LOCK_TYPE             ||= "lock_type"
-  ON_CLIENT_CONFLICT    ||= "on_client_conflict"
-  ON_CONFLICT           ||= "on_conflict"
-  ON_SERVER_CONFLICT    ||= "on_server_conflict"
-  PAYLOAD               ||= "payload"
-  PROCESSES             ||= "processes"
-  QUEUE                 ||= "queue"
-  RETRY                 ||= "retry"
-  SCHEDULE              ||= "schedule"
-  TIME                  ||= "time"
-  TIMEOUT               ||= "timeout"
-  TTL                   ||= "ttl"
-  TYPE                  ||= "type"
-  UNIQUE                ||= "unique"
-  UNIQUE_ACROSS_QUEUES  ||= "unique_across_queues"
-  UNIQUE_ACROSS_WORKERS ||= "unique_across_workers"
-  UNIQUE_ARGS           ||= "unique_args"
-  UNIQUE_ARGS_METHOD    ||= "unique_args_method"
-  UNIQUE_DIGEST         ||= "unique_digest"
-  UNIQUE_PREFIX         ||= "unique_prefix"
-  UNIQUE_REAPER         ||= "uniquejobs:reaper"
-  WORKER                ||= "worker"
+  ARGS                  = "args"
+  APARTMENT             = "apartment"
+  AT                    = "at"
+  CHANGELOGS            = "uniquejobs:changelog"
+  CLASS                 = "class"
+  CREATED_AT            = "created_at"
+  DEAD_VERSION          = "uniquejobs:dead"
+  DIGESTS               = "uniquejobs:digests"
+  EXPIRING_DIGESTS      = "uniquejobs:expiring_digests"
+  ERRORS                = "errors"
+  JID                   = "jid"
+  LIMIT                 = "limit"
+  LIVE_VERSION          = "uniquejobs:live"
+  LOCK                  = "lock"
+  LOCK_ARGS             = "lock_args"
+  LOCK_ARGS_METHOD      = "lock_args_method"
+  LOCK_DIGEST           = "lock_digest"
+  LOCK_EXPIRATION       = "lock_expiration"
+  LOCK_INFO             = "lock_info"
+  LOCK_LIMIT            = "lock_limit"
+  LOCK_PREFIX           = "lock_prefix"
+  LOCK_TIMEOUT          = "lock_timeout"
+  LOCK_TTL              = "lock_ttl"
+  LOCK_TYPE             = "lock_type"
+  ON_CLIENT_CONFLICT    = "on_client_conflict"
+  ON_CONFLICT           = "on_conflict"
+  ON_SERVER_CONFLICT    = "on_server_conflict"
+  PAYLOAD               = "payload"
+  PROCESSES             = "processes"
+  QUEUE                 = "queue"
+  RETRY                 = "retry"
+  SCHEDULE              = "schedule"
+  TIME                  = "time"
+  TIMEOUT               = "timeout"
+  TTL                   = "ttl"
+  TYPE                  = "type"
+  UNIQUE                = "unique"
+  UNIQUE_ACROSS_QUEUES  = "unique_across_queues"
+  UNIQUE_ACROSS_WORKERS = "unique_across_workers"
+  UNIQUE_ARGS           = "unique_args"
+  UNIQUE_ARGS_METHOD    = "unique_args_method"
+  UNIQUE_DIGEST         = "unique_digest"
+  UNIQUE_PREFIX         = "unique_prefix"
+  UNIQUE_REAPER         = "uniquejobs:reaper"
+  WORKER                = "worker"
 end

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -98,7 +98,7 @@ class Hash
     def _deep_transform_keys_in_object(object, &block)
       case object
       when Hash
-        object.each_with_object({}) do |(key, value), result|
+        object.each_with_object(self.class.new) do |(key, value), result|
           result[yield(key)] = _deep_transform_keys_in_object(value, &block)
         end
       when Array

data/lib/sidekiq_unique_jobs/deprecation.rb

@@ -7,6 +7,13 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   class Deprecation
+    #
+    # Mute warnings from this gem in a threaded context
+    #
+    #
+    # @return [void] <description>
+    #
+    # @yieldreturn [void]
     def self.muted
       orig_val = Thread.current[:uniquejobs_mute_deprecations]
       Thread.current[:uniquejobs_mute_deprecations] = true
@@ -15,21 +22,44 @@ module SidekiqUniqueJobs
       Thread.current[:uniquejobs_mute_deprecations] = orig_val
     end
 
+    #
+    # Check if deprecation warnings have been muted
+    #
+    #
+    # @return [true,false]
+    #
     def self.muted?
       Thread.current[:uniquejobs_mute_deprecations] == true
     end
 
+    #
+    # Warn about deprecation
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
     def self.warn(msg)
       return if SidekiqUniqueJobs::Deprecation.muted?
 
       warn "DEPRECATION WARNING: #{msg}"
+      nil
     end
 
+    #
+    # Warn about deprecation and provide a context
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
     def self.warn_with_backtrace(msg)
       return if SidekiqUniqueJobs::Deprecation.muted?
 
       trace = "\n\nCALLED FROM:\n#{caller.join("\n")}"
       warn(msg + trace)
+
+      nil
     end
   end
 end

data/lib/sidekiq_unique_jobs/digests.rb

@@ -14,8 +14,8 @@ module SidekiqUniqueJobs
     # @return [String] the default pattern to use for matching
     SCAN_PATTERN  = "*"
 
-    def initialize
-      super(DIGESTS)
+    def initialize(digests_key = DIGESTS)
+      super(digests_key)
     end
 
     #
@@ -31,7 +31,8 @@ module SidekiqUniqueJobs
     #
     # @param [String] pattern a key pattern to match with
     # @param [Integer] count the maximum number
-    # @return [Array<String>] with unique digests
+    # @return [Hash<String,Float>] Hash mapping of digest matching the given pattern and score
+
     def delete_by_pattern(pattern, count: DEFAULT_COUNT)
       result, elapsed = timed do
         digests = entries(pattern: pattern, count: count).keys
@@ -80,11 +81,7 @@ module SidekiqUniqueJobs
       options[:match] = pattern
       options[:count] = count
 
-      result = redis { |conn| conn.zscan_each(key, **options).to_a }
-
-      result.each_with_object({}) do |entry, hash|
-        hash[entry[0]] = entry[1]
-      end
+      redis { |conn| conn.zscan_each(key, **options).to_a }.to_h
     end
 
     #
@@ -98,9 +95,9 @@ module SidekiqUniqueJobs
     #
     def page(cursor: 0, pattern: SCAN_PATTERN, page_size: 100)
       redis do |conn|
-        total_size, digests = conn.multi do
-          conn.zcard(key)
-          conn.zscan(key, cursor, match: pattern, count: page_size)
+        total_size, digests = conn.multi do |pipeline|
+          pipeline.zcard(key)
+          pipeline.zscan(key, cursor, match: pattern, count: page_size)
         end
 
         [

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -22,6 +22,7 @@ module SidekiqUniqueJobs
   # Raised when no block was given
   #
   class NoBlockGiven < SidekiqUniqueJobs::UniqueJobsError; end
+
   #
   # Raised when a notification has been mistyped
   #
@@ -71,14 +72,14 @@ module SidekiqUniqueJobs
   class InvalidUniqueArguments < UniqueJobsError
     def initialize(options)
       given            = options[:given]
-      worker_class     = options[:worker_class]
+      job_class        = options[:job_class]
       lock_args_method = options[:lock_args_method]
-      lock_args_meth   = worker_class.method(lock_args_method)
+      lock_args_meth   = job_class.method(lock_args_method)
       num_args         = lock_args_meth.arity
       source_location  = lock_args_meth.source_location
 
       super(
-        "#{worker_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
+        "#{job_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
         "\n\n" \
         "   #{source_location.join(':')}"
       )

data/lib/sidekiq_unique_jobs/expiring_digests.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class ExpiringDigests provides access to the expiring digests used by until_expired locks
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class ExpiringDigests < Digests
+    def initialize
+      super(EXPIRING_DIGESTS)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/job.rb

@@ -11,6 +11,7 @@ module SidekiqUniqueJobs
     # @return [Hash] the job hash
     def prepare(item)
       stringify_on_conflict_hash(item)
+      add_lock_type(item)
       add_lock_timeout(item)
       add_lock_ttl(item)
       add_digest(item)
@@ -54,5 +55,9 @@ module SidekiqUniqueJobs
     def add_lock_prefix(item)
       item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.lock_prefix
     end
+
+    def add_lock_type(item)
+      item[LOCK] ||= SidekiqUniqueJobs::LockType.call(item)
+    end
   end
 end

data/lib/sidekiq_unique_jobs/json.rb

@@ -20,6 +20,13 @@ module SidekiqUniqueJobs
       ::JSON.parse(string)
     end
 
+    #
+    # Prevents trying JSON.load from raising errors given argument is a hash
+    #
+    # @param [String, Hash] string the JSON string to parse
+    #
+    # @return [Hash,Array]
+    #
     def safe_load_json(string)
       return string if string.is_a?(Hash)
 

data/lib/sidekiq_unique_jobs/key.rb

@@ -33,6 +33,10 @@ module SidekiqUniqueJobs
     # @!attribute [r] digests
     #   @return [String] the zset with locked digests
     attr_reader :digests
+    #
+    # @!attribute [r] expiring_digests
+    #   @return [String] the zset with locked expiring_digests
+    attr_reader :expiring_digests
 
     #
     # Initialize a new Key
@@ -40,13 +44,14 @@ module SidekiqUniqueJobs
     # @param [String] digest the digest to use as key
     #
     def initialize(digest)
-      @digest    = digest
-      @queued    = suffixed_key("QUEUED")
-      @primed    = suffixed_key("PRIMED")
-      @locked    = suffixed_key("LOCKED")
-      @info      = suffixed_key("INFO")
-      @changelog = CHANGELOGS
-      @digests   = DIGESTS
+      @digest           = digest
+      @queued           = suffixed_key("QUEUED")
+      @primed           = suffixed_key("PRIMED")
+      @locked           = suffixed_key("LOCKED")
+      @info             = suffixed_key("INFO")
+      @changelog        = CHANGELOGS
+      @digests          = DIGESTS
+      @expiring_digests = EXPIRING_DIGESTS
     end
 
     #
@@ -81,7 +86,7 @@ module SidekiqUniqueJobs
     # @return [Array] an ordered array with all keys
     #
     def to_a
-      [digest, queued, primed, locked, info, changelog, digests]
+      [digest, queued, primed, locked, info, changelog, digests, expiring_digests]
     end
 
     private

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -91,6 +91,13 @@ module SidekiqUniqueJobs
       #   @return [Integer] the current locking attempt
       attr_reader :attempt
 
+      #
+      # Eases testing by allowing the lock implementation to add the missing
+      # keys to the job hash.
+      #
+      #
+      # @return [void] the return value should be irrelevant
+      #
       def prepare_item
         return if item.key?(LOCK_DIGEST)
 
@@ -100,34 +107,22 @@ module SidekiqUniqueJobs
       end
 
       #
-      # Handle when lock failed
+      # Call whatever strategry that has been configured
       #
-      # @param [Symbol] location: :client or :server
+      # @param [Symbol] origin: the origin `:client` or `:server`
       #
-      # @return [void]
+      # @return [void] the return value is irrelevant
       #
-      def lock_failed(origin: :client)
-        reflect(:lock_failed, item)
-        call_strategy(origin: origin)
-        nil
-      end
-
+      # @yieldparam [void] if a new job id was set and a block is given
+      # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
+      #   one specific situation to yield back to the middleware.
       def call_strategy(origin:)
-        @attempt += 1
+        new_job_id = nil
+        strategy   = strategy_for(origin)
+        @attempt  += 1
 
-        case origin
-        when :client
-          client_strategy.call { lock if replace? }
-        when :server
-          server_strategy.call { lock if replace? }
-        else
-          raise SidekiqUniqueJobs::InvalidArgument,
-                "either `for: :server` or `for: :client` needs to be specified"
-        end
-      end
-
-      def replace?
-        client_strategy.replace? && attempt < 2
+        strategy.call { new_job_id = lock if strategy.replace? && @attempt < 2 }
+        yield if new_job_id && block_given?
       end
 
       def unlock_and_callback
@@ -144,6 +139,18 @@ module SidekiqUniqueJobs
         raise
       end
 
+      def strategy_for(origin)
+        case origin
+        when :client
+          client_strategy
+        when :server
+          server_strategy
+        else
+          raise SidekiqUniqueJobs::InvalidArgument,
+                "#origin needs to be either `:server` or `:client`"
+        end
+      end
+
       def client_strategy
         @client_strategy ||=
           OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -22,9 +22,15 @@ module SidekiqUniqueJobs
       #
       # @yield to the caller when given a block
       #
-      def lock(origin: :client)
-        return lock_failed(origin: origin) unless (token = locksmith.lock)
-        return yield token if block_given?
+      def lock(origin: :client, &block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: origin, &block)
+
+          return
+        end
+
+        yield if block
 
         token
       end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -17,9 +17,15 @@ module SidekiqUniqueJobs
       #
       # @yield to the caller when given a block
       #
-      def lock
-        return lock_failed(origin: :client) unless (token = locksmith.lock)
-        return yield token if block_given?
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
+
+          return
+        end
+
+        yield if block
 
         token
       end
@@ -27,10 +33,15 @@ module SidekiqUniqueJobs
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        locksmith.execute do
+        executed = locksmith.execute do
           yield
+        ensure
           unlock_and_callback
         end
+
+        reflect(:execution_failed, item) unless executed
+
+        nil
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -15,11 +15,17 @@ module SidekiqUniqueJobs
       #
       # @return [String, nil] the locked jid when properly locked, else nil.
       #
-      def lock
-        return lock_failed unless (job_id = locksmith.lock)
-        return yield job_id if block_given?
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
 
-        job_id
+          return
+        end
+
+        yield if block
+
+        token
       end
 
       # Executes in the Sidekiq server process
@@ -27,6 +33,10 @@ module SidekiqUniqueJobs
       def execute
         callback_safely if locksmith.unlock
         yield
+      rescue StandardError => ex
+        reflect(:execution_failed, item, ex)
+        locksmith.lock(wait: 1)
+        raise
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -17,17 +17,25 @@ module SidekiqUniqueJobs
       #
       # @yield to the caller when given a block
       #
-      def lock
-        return lock_failed unless (job_id = locksmith.lock)
-        return yield job_id if block_given?
+      def lock(&block)
+        unless (token = locksmith.lock)
+          reflect(:lock_failed, item)
+          call_strategy(origin: :client, &block)
 
-        job_id
+          return
+        end
+
+        yield if block
+
+        token
       end
 
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute(&block)
-        locksmith.execute(&block)
+        executed = locksmith.execute(&block)
+
+        reflect(:execution_failed, item) unless executed
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -11,7 +11,7 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class WhileExecuting < BaseLock
-      RUN_SUFFIX ||= ":RUN"
+      RUN_SUFFIX = ":RUN"
 
       include SidekiqUniqueJobs::OptionsWithFallback
       include SidekiqUniqueJobs::Logging::Middleware
@@ -30,7 +30,7 @@ module SidekiqUniqueJobs
       # @return [true] always returns true
       def lock
         job_id = item[JID]
-        yield job_id if block_given?
+        yield if block_given?
 
         job_id
       end
@@ -38,13 +38,17 @@ module SidekiqUniqueJobs
       # Executes in the Sidekiq server process.
       #   These jobs are locked in the server process not from the client
       # @yield to the worker class perform method
-      def execute
+      def execute(&block)
         with_logging_context do
-          call_strategy(origin: :server) unless locksmith.execute do
+          executed = locksmith.execute do
             yield
-            callback_safely if locksmith.unlock
           ensure
-            locksmith.unlock
+            unlock_and_callback
+          end
+
+          unless executed
+            reflect(:execution_failed, item)
+            call_strategy(origin: :server, &block)
           end
         end
       end