sidekiq-unique-jobs 6.0.25 → 7.1.29

data/lib/sidekiq_unique_jobs/constants.rb

@@ -3,31 +3,52 @@
 #
 # Module with constants to avoid string duplication
 #
-# @author Mikael Henriksson <mikael@zoolutions.se>
+# @author Mikael Henriksson <mikael@mhenrixon.com>
 #
 module SidekiqUniqueJobs
-  ARGS_KEY                  ||= "args"
-  APARTMENT                 ||= "apartment"
-  AT_KEY                    ||= "at"
-  CLASS_KEY                 ||= "class"
-  JAVA                      ||= "java"
-  JID_KEY                   ||= "jid"
-  LOCK_DIGEST_KEY           ||= "lock_digest"
-  LOCK_EXPIRATION_KEY       ||= "lock_expiration"
-  LOCK_TIMEOUT_KEY          ||= "lock_timeout"
-  LOCK_TTL_KEY              ||= "lock_ttl"
-  LOG_DUPLICATE_KEY         ||= "log_duplicate_payload"
-  QUEUE_KEY                 ||= "queue"
-  UNIQUE_ACROSS_QUEUES_KEY  ||= "unique_across_queues"
-  UNIQUE_ACROSS_WORKERS_KEY ||= "unique_across_workers"
-  UNIQUE_ARGS_KEY           ||= "unique_args"
-  UNIQUE_DIGEST_KEY         ||= "unique_digest"
-  UNIQUE_KEY                ||= "unique"
-  UNIQUE_SET                ||= "unique:keys"
-  LOCK_KEY                  ||= "lock"
-  ON_CONFLICT_KEY           ||= "on_conflict"
-  UNIQUE_ON_ALL_QUEUES_KEY  ||= "unique_on_all_queues" # TODO: Remove in v6.1
-  UNIQUE_PREFIX_KEY         ||= "unique_prefix"
-  RETRY_SET                 ||= "retry"
-  SCHEDULE_SET              ||= "schedule"
+  ARGS                  = "args"
+  APARTMENT             = "apartment"
+  AT                    = "at"
+  CHANGELOGS            = "uniquejobs:changelog"
+  CLASS                 = "class"
+  CREATED_AT            = "created_at"
+  DEAD_VERSION          = "uniquejobs:dead"
+  DIGESTS               = "uniquejobs:digests"
+  EXPIRING_DIGESTS      = "uniquejobs:expiring_digests"
+  ERRORS                = "errors"
+  JID                   = "jid"
+  LIMIT                 = "limit"
+  LIVE_VERSION          = "uniquejobs:live"
+  LOCK                  = "lock"
+  LOCK_ARGS             = "lock_args"
+  LOCK_ARGS_METHOD      = "lock_args_method"
+  LOCK_DIGEST           = "lock_digest"
+  LOCK_EXPIRATION       = "lock_expiration"
+  LOCK_INFO             = "lock_info"
+  LOCK_LIMIT            = "lock_limit"
+  LOCK_PREFIX           = "lock_prefix"
+  LOCK_TIMEOUT          = "lock_timeout"
+  LOCK_TTL              = "lock_ttl"
+  LOCK_TYPE             = "lock_type"
+  ON_CLIENT_CONFLICT    = "on_client_conflict"
+  ON_CONFLICT           = "on_conflict"
+  ON_SERVER_CONFLICT    = "on_server_conflict"
+  PAYLOAD               = "payload"
+  PROCESSES             = "processes"
+  QUEUE                 = "queue"
+  RETRY                 = "retry"
+  SCHEDULE              = "schedule"
+  TIME                  = "time"
+  TIMEOUT               = "timeout"
+  TTL                   = "ttl"
+  TYPE                  = "type"
+  UNIQUE                = "unique"
+  UNIQUE_ACROSS_QUEUES  = "unique_across_queues"
+  UNIQUE_ACROSS_WORKERS = "unique_across_workers"
+  UNIQUE_ARGS           = "unique_args"
+  UNIQUE_ARGS_METHOD    = "unique_args_method"
+  UNIQUE_DIGEST         = "unique_digest"
+  UNIQUE_PREFIX         = "unique_prefix"
+  UNIQUE_REAPER         = "uniquejobs:reaper"
+  WORKER                = "worker"
 end

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -2,21 +2,67 @@
 
 # :nocov:
 
+#
+# Monkey patches for the ruby Hash
+#
 class Hash
   unless {}.respond_to?(:slice)
+    #
+    # Returns only the matching keys in a new hash
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
     end
   end
 
+  unless {}.respond_to?(:deep_stringify_keys)
+    #
+    # Depp converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_stringify_keys
+      deep_transform_keys(&:to_s)
+    end
+  end
+
+  unless {}.respond_to?(:deep_transform_keys)
+    #
+    # Deep transfor all keys by yielding to the caller
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_transform_keys(&block)
+      _deep_transform_keys_in_object(self, &block)
+    end
+  end
+
   unless {}.respond_to?(:stringify_keys)
+    #
+    # Converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
     def stringify_keys
       transform_keys(&:to_s)
     end
   end
 
   unless {}.respond_to?(:transform_keys)
+    #
+    # Transforms all keys by yielding to the caller
+    #
+    #
+    # @return [Hash]
+    #
     def transform_keys
       result = {}
       each_key do |key|
@@ -27,6 +73,13 @@ class Hash
   end
 
   unless {}.respond_to?(:slice!)
+    #
+    # Removes all keys not provided from the current hash and returns it
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice!(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       omit = slice(*self.keys - keys)
@@ -37,10 +90,37 @@ class Hash
       omit
     end
   end
+
+  private
+
+  unless {}.respond_to?(:_deep_transform_keys_in_object)
+    # support methods for deep transforming nested hashes and arrays
+    def _deep_transform_keys_in_object(object, &block)
+      case object
+      when Hash
+        object.each_with_object({}) do |(key, value), result|
+          result[yield(key)] = _deep_transform_keys_in_object(value, &block)
+        end
+      when Array
+        object.map { |element| _deep_transform_keys_in_object(element, &block) }
+      else
+        object
+      end
+    end
+  end
 end
 
+#
+# Monkey patches for the ruby Array
+#
 class Array
   unless [].respond_to?(:extract_options!)
+    #
+    # Extract the last argument if it is a hash
+    #
+    #
+    # @return [Hash]
+    #
     def extract_options!
       if last.is_a?(Hash) && last.instance_of?(Hash)
         pop

data/lib/sidekiq_unique_jobs/deprecation.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Deprecation provides logging of deprecations
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Deprecation
+    #
+    # Mute warnings from this gem in a threaded context
+    #
+    #
+    # @return [void] <description>
+    #
+    # @yieldreturn [void]
+    def self.muted
+      orig_val = Thread.current[:uniquejobs_mute_deprecations]
+      Thread.current[:uniquejobs_mute_deprecations] = true
+      yield
+    ensure
+      Thread.current[:uniquejobs_mute_deprecations] = orig_val
+    end
+
+    #
+    # Check if deprecation warnings have been muted
+    #
+    #
+    # @return [true,false]
+    #
+    def self.muted?
+      Thread.current[:uniquejobs_mute_deprecations] == true
+    end
+
+    #
+    # Warn about deprecation
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      warn "DEPRECATION WARNING: #{msg}"
+      nil
+    end
+
+    #
+    # Warn about deprecation and provide a context
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn_with_backtrace(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      trace = "\n\nCALLED FROM:\n#{caller.join("\n")}"
+      warn(msg + trace)
+
+      nil
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/digests.rb

@@ -1,87 +1,66 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  # Utility module to help manage unique digests in redis.
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  module Digests
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Digests < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
     DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
     SCAN_PATTERN  = "*"
-    CHUNK_SIZE    = 100
-
-    include SidekiqUniqueJobs::Logging
-    include SidekiqUniqueJobs::Connection
-    extend self
 
-    # Return unique digests matching pattern
-    #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] count the maximum number to match
-    # @return [Array<String>] with unique digests
-    def all(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
-      redis { |conn| conn.sscan_each(UNIQUE_SET, match: pattern, count: count).to_a }
+    def initialize(digests_key = DIGESTS)
+      super(digests_key)
     end
 
-    # Paginate unique digests
     #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] cursor the maximum number to match
-    # @param [Integer] page_size the current cursor position
+    # Adds a digest
     #
-    # @return [Array<String>] with unique digests
-    def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
-      redis do |conn|
-        total_size, digests = conn.multi do
-          conn.scard(UNIQUE_SET)
-          conn.sscan(UNIQUE_SET, cursor, match: pattern, count: page_size)
-        end
-
-        [total_size, digests[0], digests[1]]
-      end
-    end
-
-    # Get a total count of unique digests
+    # @param [String] digest the digest to add
     #
-    # @return [Integer] number of digests
-    def count
-      redis { |conn| conn.scard(UNIQUE_SET) }
+    def add(digest)
+      redis { |conn| conn.zadd(key, now_f, digest) }
     end
 
-    # Deletes unique digest either by a digest or pattern
+    # Deletes unique digests by pattern
     #
-    # @param [String] digest the full digest to delete
     # @param [String] pattern a key pattern to match with
     # @param [Integer] count the maximum number
-    # @raise [ArgumentError] when both pattern and digest are nil
-    # @return [Array<String>] with unique digests
-    def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
-      warn("#{self}.#{__method__} has been deprecated and will be removed in a future version")
+    # @return [Hash<String,Float>] Hash mapping of digest matching the given pattern and score
+
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = entries(pattern: pattern, count: count).keys
+        redis { |conn| BatchDelete.call(digests, conn) }
+      end
 
-      return delete_by_pattern(pattern, count: count) if pattern
-      return delete_by_digest(digest) if digest
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
 
-      raise ArgumentError, "either digest or pattern need to be provided"
+      result
     end
 
-    # Deletes unique digest either by a digest or pattern
+    # Delete unique digests by digest
+    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
     #
-    # @param [String] digest the full digest to delete
+    # @param [String] digest a unique digest to delete
     def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
       result, elapsed = timed do
-        Scripts.call(:delete_by_digest, nil, keys: [
-                       UNIQUE_SET,
-                       digest,
-                       "#{digest}:EXISTS",
-                       "#{digest}:GRABBED",
-                       "#{digest}:AVAILABLE",
-                       "#{digest}:VERSION",
-                       "#{digest}:RUN:EXISTS",
-                       "#{digest}:RUN:GRABBED",
-                       "#{digest}:RUN:AVAILABLE",
-                       "#{digest}:RUN:VERSION",
-                     ])
-
-        count
+        call_script(:delete_by_digest, [
+                      digest,
+                      "#{digest}:QUEUED",
+                      "#{digest}:PRIMED",
+                      "#{digest}:LOCKED",
+                      "#{digest}:RUN",
+                      "#{digest}:RUN:QUEUED",
+                      "#{digest}:RUN:PRIMED",
+                      "#{digest}:RUN:LOCKED",
+                      key,
+                    ])
       end
 
       log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
@@ -89,55 +68,44 @@ module SidekiqUniqueJobs
       result
     end
 
-    # Deletes unique digests by pattern
     #
-    # @param [String] pattern a key pattern to match with
-    # @param [Integer] count the maximum number
-    # @return [Array<String>] with unique digests
-    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
-      result, elapsed = timed do
-        digests = all(pattern: pattern, count: count)
-        batch_delete(digests)
-        digests.size
-      end
-
-      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+    # The entries in this sorted set
+    #
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] count DEFAULT_COUNT the number of entries to return
+    #
+    # @return [Array<String>] an array of digests matching the given pattern
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      options = {}
+      options[:match] = pattern
+      options[:count] = count
 
-      result
+      redis { |conn| conn.zscan_each(key, **options).to_a }.to_h
     end
 
-    private
-
-    def batch_delete(digests) # rubocop:disable Metrics/MethodLength
+    #
+    # Returns a paginated
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] page_size 100 the size per page
+    #
+    # @return [Array<Integer, Integer, Array<Lock>>] total_size, next_cursor, locks
+    #
+    def page(cursor: 0, pattern: SCAN_PATTERN, page_size: 100)
       redis do |conn|
-        digests.each_slice(CHUNK_SIZE) do |chunk|
-          conn.pipelined do
-            chunk.each do |digest|
-              conn.del digest
-              conn.srem(UNIQUE_SET, digest)
-              conn.del("#{digest}:EXISTS")
-              conn.del("#{digest}:GRABBED")
-              conn.del("#{digest}:VERSION")
-              conn.del("#{digest}:AVAILABLE")
-              conn.del("#{digest}:RUN:EXISTS")
-              conn.del("#{digest}:RUN:GRABBED")
-              conn.del("#{digest}:RUN:VERSION")
-              conn.del("#{digest}:RUN:AVAILABLE")
-            end
-          end
+        total_size, digests = conn.multi do |pipeline|
+          pipeline.zcard(key)
+          pipeline.zscan(key, cursor, match: pattern, count: page_size)
         end
-      end
-    end
 
-    def timed
-      start = current_time
-      result = yield
-      elapsed = (current_time - start).round(2)
-      [result, elapsed]
-    end
-
-    def current_time
-      Time.now
+        [
+          total_size.to_i,
+          digests[0].to_i, # next_cursor
+          digests[1].map { |digest, score| Lock.new(digest, time: score) }, # entries
+        ]
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -1,29 +1,105 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
+  #
+  # Base class for all exceptions raised from the gem
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UniqueJobsError < ::RuntimeError
+  end
+
   # Error raised when a Lua script fails to execute
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class Conflict < StandardError
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Conflict < UniqueJobsError
     def initialize(item)
-      super("Item with the key: #{item[UNIQUE_DIGEST_KEY]} is already scheduled or processing")
+      super("Item with the key: #{item[LOCK_DIGEST]} is already scheduled or processing")
+    end
+  end
+
+  #
+  # Raised when no block was given
+  #
+  class NoBlockGiven < SidekiqUniqueJobs::UniqueJobsError; end
+
+  #
+  # Raised when a notification has been mistyped
+  #
+  class NoSuchNotificationError < UniqueJobsError; end
+
+  #
+  # Error raised when trying to add a duplicate lock
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateLock < UniqueJobsError
+  end
+
+  #
+  # Error raised when trying to add a duplicate stragegy
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateStrategy < UniqueJobsError
+  end
+
+  #
+  # Error raised when an invalid argument is given
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidArgument < UniqueJobsError
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidWorker < UniqueJobsError
+    def initialize(lock_config)
+      super(<<~FAILURE_MESSAGE)
+        Expected #{lock_config.worker} to have valid sidekiq options but found the following problems:
+        #{lock_config.errors_as_string}
+      FAILURE_MESSAGE
+    end
+  end
+
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class InvalidUniqueArguments < UniqueJobsError
+    def initialize(options)
+      given            = options[:given]
+      job_class        = options[:job_class]
+      lock_args_method = options[:lock_args_method]
+      lock_args_meth   = job_class.method(lock_args_method)
+      num_args         = lock_args_meth.arity
+      source_location  = lock_args_meth.source_location
+
+      super(
+        "#{job_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
+        "\n\n" \
+        "   #{source_location.join(':')}"
+      )
     end
   end
 
-  # Error raised from {OnConflict::Raise}
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class ScriptError < StandardError
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Redis::CommandError] source_exception exception to handle
-    def initialize(file_name:, source_exception:)
-      super("Problem compiling #{file_name}. Message: #{source_exception.message}")
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class NotUniqueWorker < UniqueJobsError
+    def initialize(options)
+      super("#{options[:class]} is not configured for uniqueness. Missing the key `:lock` in #{options.inspect}")
     end
   end
 
   # Error raised from {OptionsWithFallback#lock_class}
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class UnknownLock < StandardError
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class UnknownLock < UniqueJobsError
   end
 end

data/lib/sidekiq_unique_jobs/expiring_digests.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class ExpiringDigests provides access to the expiring digests used by until_expired locks
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class ExpiringDigests < Digests
+    def initialize
+      super(EXPIRING_DIGESTS)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/job.rb

@@ -3,27 +3,56 @@
 module SidekiqUniqueJobs
   # Utility class to append uniqueness to the sidekiq job hash
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Job
     extend self
 
-    # Adds timeout, expiration, unique_args, unique_prefix and unique_digest to the sidekiq job hash
-    # @return [void] nothing returned here matters
-    def add_uniqueness(item)
-      add_timeout_and_expiration(item)
-      add_unique_args_and_digest(item)
+    # Adds timeout, expiration, lock_args, lock_prefix and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def prepare(item)
+      stringify_on_conflict_hash(item)
+      add_lock_timeout(item)
+      add_lock_ttl(item)
+      add_digest(item)
+    end
+
+    # Adds lock_args, lock_prefix and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def add_digest(item)
+      add_lock_prefix(item)
+      add_lock_args(item)
+      add_lock_digest(item)
+
+      item
     end
 
     private
 
-    def add_timeout_and_expiration(item)
-      calculator                = SidekiqUniqueJobs::Timeout::Calculator.new(item)
-      item[LOCK_TIMEOUT_KEY]    = calculator.lock_timeout
-      item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
+    def stringify_on_conflict_hash(item)
+      on_conflict = item[ON_CONFLICT]
+      return unless on_conflict.is_a?(Hash)
+
+      item[ON_CONFLICT] = on_conflict.deep_stringify_keys
+    end
+
+    def add_lock_ttl(item)
+      item[LOCK_TTL] = SidekiqUniqueJobs::LockTTL.calculate(item)
+    end
+
+    def add_lock_timeout(item)
+      item[LOCK_TIMEOUT] ||= SidekiqUniqueJobs::LockTimeout.calculate(item)
+    end
+
+    def add_lock_args(item)
+      item[LOCK_ARGS] ||= SidekiqUniqueJobs::LockArgs.call(item)
+    end
+
+    def add_lock_digest(item)
+      item[LOCK_DIGEST] ||= SidekiqUniqueJobs::LockDigest.call(item)
     end
 
-    def add_unique_args_and_digest(item)
-      SidekiqUniqueJobs::UniqueArgs.digest(item)
+    def add_lock_prefix(item)
+      item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.lock_prefix
     end
   end
 end