sidekiq-unique-jobs 6.0.25 → 7.1.5

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -2,21 +2,67 @@
 
 # :nocov:
 
+#
+# Monkey patches for the ruby Hash
+#
 class Hash
   unless {}.respond_to?(:slice)
+    #
+    # Returns only the matching keys in a new hash
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
     end
   end
 
+  unless {}.respond_to?(:deep_stringify_keys)
+    #
+    # Depp converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_stringify_keys
+      deep_transform_keys(&:to_s)
+    end
+  end
+
+  unless {}.respond_to?(:deep_transform_keys)
+    #
+    # Deep transfor all keys by yielding to the caller
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_transform_keys(&block)
+      _deep_transform_keys_in_object(self, &block)
+    end
+  end
+
   unless {}.respond_to?(:stringify_keys)
+    #
+    # Converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
     def stringify_keys
       transform_keys(&:to_s)
     end
   end
 
   unless {}.respond_to?(:transform_keys)
+    #
+    # Transforms all keys by yielding to the caller
+    #
+    #
+    # @return [Hash]
+    #
     def transform_keys
       result = {}
       each_key do |key|
@@ -27,6 +73,13 @@ class Hash
   end
 
   unless {}.respond_to?(:slice!)
+    #
+    # Removes all keys not provided from the current hash and returns it
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice!(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       omit = slice(*self.keys - keys)
@@ -37,10 +90,37 @@ class Hash
       omit
     end
   end
+
+  private
+
+  unless {}.respond_to?(:_deep_transform_keys_in_object)
+    # support methods for deep transforming nested hashes and arrays
+    def _deep_transform_keys_in_object(object, &block)
+      case object
+      when Hash
+        object.each_with_object({}) do |(key, value), result|
+          result[yield(key)] = _deep_transform_keys_in_object(value, &block)
+        end
+      when Array
+        object.map { |element| _deep_transform_keys_in_object(element, &block) }
+      else
+        object
+      end
+    end
+  end
 end
 
+#
+# Monkey patches for the ruby Array
+#
 class Array
   unless [].respond_to?(:extract_options!)
+    #
+    # Extract the last argument if it is a hash
+    #
+    #
+    # @return [Hash]
+    #
     def extract_options!
       if last.is_a?(Hash) && last.instance_of?(Hash)
         pop

data/lib/sidekiq_unique_jobs/deprecation.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Deprecation provides logging of deprecations
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Deprecation
+    def self.muted
+      orig_val = Thread.current[:uniquejobs_mute_deprecations]
+      Thread.current[:uniquejobs_mute_deprecations] = true
+      yield
+    ensure
+      Thread.current[:uniquejobs_mute_deprecations] = orig_val
+    end
+
+    def self.muted?
+      Thread.current[:uniquejobs_mute_deprecations] == true
+    end
+
+    def self.warn(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      warn "DEPRECATION WARNING: #{msg}"
+    end
+
+    def self.warn_with_backtrace(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      trace = "\n\nCALLED FROM:\n#{caller.join("\n")}"
+      warn(msg + trace)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/digests.rb

@@ -1,87 +1,65 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  # Utility module to help manage unique digests in redis.
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  module Digests
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Digests < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
     DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
     SCAN_PATTERN  = "*"
-    CHUNK_SIZE    = 100
-
-    include SidekiqUniqueJobs::Logging
-    include SidekiqUniqueJobs::Connection
-    extend self
 
-    # Return unique digests matching pattern
-    #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] count the maximum number to match
-    # @return [Array<String>] with unique digests
-    def all(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
-      redis { |conn| conn.sscan_each(UNIQUE_SET, match: pattern, count: count).to_a }
+    def initialize
+      super(DIGESTS)
     end
 
-    # Paginate unique digests
     #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] cursor the maximum number to match
-    # @param [Integer] page_size the current cursor position
+    # Adds a digest
     #
-    # @return [Array<String>] with unique digests
-    def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
-      redis do |conn|
-        total_size, digests = conn.multi do
-          conn.scard(UNIQUE_SET)
-          conn.sscan(UNIQUE_SET, cursor, match: pattern, count: page_size)
-        end
-
-        [total_size, digests[0], digests[1]]
-      end
-    end
-
-    # Get a total count of unique digests
+    # @param [String] digest the digest to add
     #
-    # @return [Integer] number of digests
-    def count
-      redis { |conn| conn.scard(UNIQUE_SET) }
+    def add(digest)
+      redis { |conn| conn.zadd(key, now_f, digest) }
     end
 
-    # Deletes unique digest either by a digest or pattern
+    # Deletes unique digests by pattern
     #
-    # @param [String] digest the full digest to delete
     # @param [String] pattern a key pattern to match with
     # @param [Integer] count the maximum number
-    # @raise [ArgumentError] when both pattern and digest are nil
     # @return [Array<String>] with unique digests
-    def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
-      warn("#{self}.#{__method__} has been deprecated and will be removed in a future version")
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = entries(pattern: pattern, count: count).keys
+        redis { |conn| BatchDelete.call(digests, conn) }
+      end
 
-      return delete_by_pattern(pattern, count: count) if pattern
-      return delete_by_digest(digest) if digest
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
 
-      raise ArgumentError, "either digest or pattern need to be provided"
+      result
     end
 
-    # Deletes unique digest either by a digest or pattern
+    # Delete unique digests by digest
+    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
     #
-    # @param [String] digest the full digest to delete
+    # @param [String] digest a unique digest to delete
     def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
       result, elapsed = timed do
-        Scripts.call(:delete_by_digest, nil, keys: [
-                       UNIQUE_SET,
-                       digest,
-                       "#{digest}:EXISTS",
-                       "#{digest}:GRABBED",
-                       "#{digest}:AVAILABLE",
-                       "#{digest}:VERSION",
-                       "#{digest}:RUN:EXISTS",
-                       "#{digest}:RUN:GRABBED",
-                       "#{digest}:RUN:AVAILABLE",
-                       "#{digest}:RUN:VERSION",
-                     ])
-
-        count
+        call_script(:delete_by_digest, [
+                      digest,
+                      "#{digest}:QUEUED",
+                      "#{digest}:PRIMED",
+                      "#{digest}:LOCKED",
+                      "#{digest}:RUN",
+                      "#{digest}:RUN:QUEUED",
+                      "#{digest}:RUN:PRIMED",
+                      "#{digest}:RUN:LOCKED",
+                      key,
+                    ])
       end
 
       log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
@@ -89,55 +67,48 @@ module SidekiqUniqueJobs
       result
     end
 
-    # Deletes unique digests by pattern
     #
-    # @param [String] pattern a key pattern to match with
-    # @param [Integer] count the maximum number
-    # @return [Array<String>] with unique digests
-    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
-      result, elapsed = timed do
-        digests = all(pattern: pattern, count: count)
-        batch_delete(digests)
-        digests.size
-      end
+    # The entries in this sorted set
+    #
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] count DEFAULT_COUNT the number of entries to return
+    #
+    # @return [Array<String>] an array of digests matching the given pattern
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      options = {}
+      options[:match] = pattern
+      options[:count] = count
 
-      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+      result = redis { |conn| conn.zscan_each(key, **options).to_a }
 
-      result
+      result.each_with_object({}) do |entry, hash|
+        hash[entry[0]] = entry[1]
+      end
     end
 
-    private
-
-    def batch_delete(digests) # rubocop:disable Metrics/MethodLength
+    #
+    # Returns a paginated
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] page_size 100 the size per page
+    #
+    # @return [Array<Integer, Integer, Array<Lock>>] total_size, next_cursor, locks
+    #
+    def page(cursor: 0, pattern: SCAN_PATTERN, page_size: 100)
       redis do |conn|
-        digests.each_slice(CHUNK_SIZE) do |chunk|
-          conn.pipelined do
-            chunk.each do |digest|
-              conn.del digest
-              conn.srem(UNIQUE_SET, digest)
-              conn.del("#{digest}:EXISTS")
-              conn.del("#{digest}:GRABBED")
-              conn.del("#{digest}:VERSION")
-              conn.del("#{digest}:AVAILABLE")
-              conn.del("#{digest}:RUN:EXISTS")
-              conn.del("#{digest}:RUN:GRABBED")
-              conn.del("#{digest}:RUN:VERSION")
-              conn.del("#{digest}:RUN:AVAILABLE")
-            end
-          end
+        total_size, digests = conn.multi do
+          conn.zcard(key)
+          conn.zscan(key, cursor, match: pattern, count: page_size)
         end
-      end
-    end
 
-    def timed
-      start = current_time
-      result = yield
-      elapsed = (current_time - start).round(2)
-      [result, elapsed]
-    end
-
-    def current_time
-      Time.now
+        [
+          total_size.to_i,
+          digests[0].to_i, # next_cursor
+          digests[1].map { |digest, score| Lock.new(digest, time: score) }, # entries
+        ]
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -1,29 +1,104 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
+  #
+  # Base class for all exceptions raised from the gem
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UniqueJobsError < ::RuntimeError
+  end
+
   # Error raised when a Lua script fails to execute
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class Conflict < StandardError
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Conflict < UniqueJobsError
     def initialize(item)
-      super("Item with the key: #{item[UNIQUE_DIGEST_KEY]} is already scheduled or processing")
+      super("Item with the key: #{item[LOCK_DIGEST]} is already scheduled or processing")
     end
   end
 
-  # Error raised from {OnConflict::Raise}
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class ScriptError < StandardError
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Redis::CommandError] source_exception exception to handle
-    def initialize(file_name:, source_exception:)
-      super("Problem compiling #{file_name}. Message: #{source_exception.message}")
+  # Raised when no block was given
+  #
+  class NoBlockGiven < SidekiqUniqueJobs::UniqueJobsError; end
+  #
+  # Raised when a notification has been mistyped
+  #
+  class NoSuchNotificationError < UniqueJobsError; end
+
+  #
+  # Error raised when trying to add a duplicate lock
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateLock < UniqueJobsError
+  end
+
+  #
+  # Error raised when trying to add a duplicate stragegy
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateStrategy < UniqueJobsError
+  end
+
+  #
+  # Error raised when an invalid argument is given
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidArgument < UniqueJobsError
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidWorker < UniqueJobsError
+    def initialize(lock_config)
+      super(<<~FAILURE_MESSAGE)
+        Expected #{lock_config.worker} to have valid sidekiq options but found the following problems:
+        #{lock_config.errors_as_string}
+      FAILURE_MESSAGE
+    end
+  end
+
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class InvalidUniqueArguments < UniqueJobsError
+    def initialize(options)
+      given            = options[:given]
+      worker_class     = options[:worker_class]
+      lock_args_method = options[:lock_args_method]
+      lock_args_meth   = worker_class.method(lock_args_method)
+      num_args         = lock_args_meth.arity
+      source_location  = lock_args_meth.source_location
+
+      super(
+        "#{worker_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
+        "\n\n" \
+        "   #{source_location.join(':')}"
+      )
+    end
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class NotUniqueWorker < UniqueJobsError
+    def initialize(options)
+      super("#{options[:class]} is not configured for uniqueness. Missing the key `:lock` in #{options.inspect}")
     end
   end
 
   # Error raised from {OptionsWithFallback#lock_class}
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class UnknownLock < StandardError
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class UnknownLock < UniqueJobsError
   end
 end

data/lib/sidekiq_unique_jobs/job.rb

@@ -3,27 +3,56 @@
 module SidekiqUniqueJobs
   # Utility class to append uniqueness to the sidekiq job hash
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Job
     extend self
 
-    # Adds timeout, expiration, unique_args, unique_prefix and unique_digest to the sidekiq job hash
-    # @return [void] nothing returned here matters
-    def add_uniqueness(item)
-      add_timeout_and_expiration(item)
-      add_unique_args_and_digest(item)
+    # Adds timeout, expiration, lock_args, lock_prefix and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def prepare(item)
+      stringify_on_conflict_hash(item)
+      add_lock_timeout(item)
+      add_lock_ttl(item)
+      add_digest(item)
+    end
+
+    # Adds lock_args, lock_prefix and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def add_digest(item)
+      add_lock_prefix(item)
+      add_lock_args(item)
+      add_lock_digest(item)
+
+      item
     end
 
     private
 
-    def add_timeout_and_expiration(item)
-      calculator                = SidekiqUniqueJobs::Timeout::Calculator.new(item)
-      item[LOCK_TIMEOUT_KEY]    = calculator.lock_timeout
-      item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
+    def stringify_on_conflict_hash(item)
+      on_conflict = item[ON_CONFLICT]
+      return unless on_conflict.is_a?(Hash)
+
+      item[ON_CONFLICT] = on_conflict.deep_stringify_keys
+    end
+
+    def add_lock_ttl(item)
+      item[LOCK_TTL] = SidekiqUniqueJobs::LockTTL.calculate(item)
+    end
+
+    def add_lock_timeout(item)
+      item[LOCK_TIMEOUT] ||= SidekiqUniqueJobs::LockTimeout.calculate(item)
+    end
+
+    def add_lock_args(item)
+      item[LOCK_ARGS] ||= SidekiqUniqueJobs::LockArgs.call(item)
+    end
+
+    def add_lock_digest(item)
+      item[LOCK_DIGEST] ||= SidekiqUniqueJobs::LockDigest.call(item)
     end
 
-    def add_unique_args_and_digest(item)
-      SidekiqUniqueJobs::UniqueArgs.digest(item)
+    def add_lock_prefix(item)
+      item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.lock_prefix
     end
   end
 end

data/lib/sidekiq_unique_jobs/json.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles loading and dumping of json
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module JSON
+    module_function
+
+    #
+    # Parses a JSON string into an object
+    #
+    # @param [String] string the object to parse
+    #
+    # @return [Object]
+    #
+    def load_json(string)
+      return if string.nil? || string.empty?
+
+      ::JSON.parse(string)
+    end
+
+    def safe_load_json(string)
+      return string if string.is_a?(Hash)
+
+      load_json(string)
+    end
+
+    #
+    # Dumps an object into a JSON string
+    #
+    # @param [Object] object a JSON convertible object
+    #
+    # @return [String] a JSON string
+    #
+    def dump_json(object)
+      ::JSON.generate(object)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/key.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Key class wraps logic dealing with various lock keys
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Key
+    #
+    # @!attribute [r] digest
+    #   @return [String] the digest key for which keys are created
+    attr_reader :digest
+    #
+    # @!attribute [r] queued
+    #   @return [String] the list key with queued job_id's
+    attr_reader :queued
+    #
+    # @!attribute [r] primed
+    #   @return [String] the list key with primed job_id's
+    attr_reader :primed
+    #
+    # @!attribute [r] locked
+    #   @return [String] the hash key with locked job_id's
+    attr_reader :locked
+    #
+    # @!attribute [r] info
+    #   @return [String] information about the lock
+    attr_reader :info
+    #
+    # @!attribute [r] changelog
+    #   @return [String] the zset with changelog entries
+    attr_reader :changelog
+    #
+    # @!attribute [r] digests
+    #   @return [String] the zset with locked digests
+    attr_reader :digests
+
+    #
+    # Initialize a new Key
+    #
+    # @param [String] digest the digest to use as key
+    #
+    def initialize(digest)
+      @digest    = digest
+      @queued    = suffixed_key("QUEUED")
+      @primed    = suffixed_key("PRIMED")
+      @locked    = suffixed_key("LOCKED")
+      @info      = suffixed_key("INFO")
+      @changelog = CHANGELOGS
+      @digests   = DIGESTS
+    end
+
+    #
+    # Provides the only important information about this keys
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      digest
+    end
+
+    # @see to_s
+    def inspect
+      digest
+    end
+
+    #
+    # Compares keys by digest
+    #
+    # @param [Key] other the key to compare with
+    #
+    # @return [true, false]
+    #
+    def ==(other)
+      digest == other.digest
+    end
+
+    #
+    # Returns all keys as an ordered array
+    #
+    # @return [Array] an ordered array with all keys
+    #
+    def to_a
+      [digest, queued, primed, locked, info, changelog, digests]
+    end
+
+    private
+
+    def suffixed_key(variable)
+      "#{digest}:#{variable}"
+    end
+  end
+end