sidekiq-unique-jobs 6.0.24 → 7.0.4

data/lib/sidekiq_unique_jobs/connection.rb

@@ -3,7 +3,7 @@
 module SidekiqUniqueJobs
   # Shared module for dealing with redis connections
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Connection
     def self.included(base)
       base.send(:extend, self)
@@ -11,11 +11,12 @@ module SidekiqUniqueJobs
 
     # Creates a connection to redis
     # @return [Sidekiq::RedisConnection, ConnectionPool] a connection to redis
-    def redis(redis_pool = nil)
-      if redis_pool
-        redis_pool.with { |conn| yield conn }
+    def redis(r_pool = nil, &block)
+      r_pool ||= defined?(redis_pool) ? redis_pool : r_pool
+      if r_pool
+        r_pool.with(&block)
       else
-        Sidekiq.redis { |conn| yield conn }
+        Sidekiq.redis(&block)
       end
     end
   end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -3,30 +3,52 @@
 #
 # Module with constants to avoid string duplication
 #
-# @author Mikael Henriksson <mikael@zoolutions.se>
+# @author Mikael Henriksson <mikael@mhenrixon.com>
 #
 module SidekiqUniqueJobs
-  ARGS_KEY                  ||= "args"
-  AT_KEY                    ||= "at"
-  CLASS_KEY                 ||= "class"
-  JAVA                      ||= "java"
-  JID_KEY                   ||= "jid"
-  LOCK_DIGEST_KEY           ||= "lock_digest"
-  LOCK_EXPIRATION_KEY       ||= "lock_expiration"
-  LOCK_TIMEOUT_KEY          ||= "lock_timeout"
-  LOCK_TTL_KEY              ||= "lock_ttl"
-  LOG_DUPLICATE_KEY         ||= "log_duplicate_payload"
-  QUEUE_KEY                 ||= "queue"
-  UNIQUE_ACROSS_QUEUES_KEY  ||= "unique_across_queues"
-  UNIQUE_ACROSS_WORKERS_KEY ||= "unique_across_workers"
-  UNIQUE_ARGS_KEY           ||= "unique_args"
-  UNIQUE_DIGEST_KEY         ||= "unique_digest"
-  UNIQUE_KEY                ||= "unique"
-  UNIQUE_SET                ||= "unique:keys"
-  LOCK_KEY                  ||= "lock"
-  ON_CONFLICT_KEY           ||= "on_conflict"
-  UNIQUE_ON_ALL_QUEUES_KEY  ||= "unique_on_all_queues" # TODO: Remove in v6.1
-  UNIQUE_PREFIX_KEY         ||= "unique_prefix"
-  RETRY_SET                 ||= "retry"
-  SCHEDULE_SET              ||= "schedule"
+  ARGS                  ||= "args"
+  APARTMENT             ||= "apartment"
+  AT                    ||= "at"
+  CHANGELOGS            ||= "uniquejobs:changelog"
+  CLASS                 ||= "class"
+  CREATED_AT            ||= "created_at"
+  DEAD_VERSION          ||= "uniquejobs:dead"
+  DIGESTS               ||= "uniquejobs:digests"
+  ERRORS                ||= "errors"
+  JID                   ||= "jid"
+  LIMIT                 ||= "limit"
+  LIVE_VERSION          ||= "uniquejobs:live"
+  LOCK                  ||= "lock"
+  LOCK_ARGS             ||= "lock_args"
+  LOCK_ARGS_METHOD      ||= "lock_args_method"
+  LOCK_DIGEST           ||= "lock_digest"
+  LOCK_EXPIRATION       ||= "lock_expiration"
+  LOCK_INFO             ||= "lock_info"
+  LOCK_LIMIT            ||= "lock_limit"
+  LOCK_PREFIX           ||= "lock_prefix"
+  LOCK_TIMEOUT          ||= "lock_timeout"
+  LOCK_TTL              ||= "lock_ttl"
+  LOCK_TYPE             ||= "lock_type"
+  LOG_DUPLICATE         ||= "log_duplicate"
+  ON_CLIENT_CONFLICT    ||= "on_client_conflict"
+  ON_CONFLICT           ||= "on_conflict"
+  ON_SERVER_CONFLICT    ||= "on_server_conflict"
+  PAYLOAD               ||= "payload"
+  PROCESSES             ||= "processes"
+  QUEUE                 ||= "queue"
+  RETRY                 ||= "retry"
+  SCHEDULE              ||= "schedule"
+  TIME                  ||= "time"
+  TIMEOUT               ||= "timeout"
+  TTL                   ||= "ttl"
+  TYPE                  ||= "type"
+  UNIQUE                ||= "unique"
+  UNIQUE_ACROSS_QUEUES  ||= "unique_across_queues"
+  UNIQUE_ACROSS_WORKERS ||= "unique_across_workers"
+  UNIQUE_ARGS           ||= "unique_args"
+  UNIQUE_ARGS_METHOD    ||= "unique_args_method"
+  UNIQUE_DIGEST         ||= "unique_digest"
+  UNIQUE_PREFIX         ||= "unique_prefix"
+  UNIQUE_REAPER         ||= "uniquejobs:reaper"
+  WORKER                ||= "worker"
 end

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -2,21 +2,67 @@
 
 # :nocov:
 
+#
+# Monkey patches for the ruby Hash
+#
 class Hash
   unless {}.respond_to?(:slice)
+    #
+    # Returns only the matching keys in a new hash
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
     end
   end
 
+  unless {}.respond_to?(:deep_stringify_keys)
+    #
+    # Depp converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_stringify_keys
+      deep_transform_keys(&:to_s)
+    end
+  end
+
+  unless {}.respond_to?(:deep_transform_keys)
+    #
+    # Deep transfor all keys by yielding to the caller
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_transform_keys(&block)
+      _deep_transform_keys_in_object(self, &block)
+    end
+  end
+
   unless {}.respond_to?(:stringify_keys)
+    #
+    # Converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
     def stringify_keys
       transform_keys(&:to_s)
     end
   end
 
   unless {}.respond_to?(:transform_keys)
+    #
+    # Transforms all keys by yielding to the caller
+    #
+    #
+    # @return [Hash]
+    #
     def transform_keys
       result = {}
       each_key do |key|
@@ -27,6 +73,13 @@ class Hash
   end
 
   unless {}.respond_to?(:slice!)
+    #
+    # Removes all keys not provided from the current hash and returns it
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice!(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       omit = slice(*self.keys - keys)
@@ -37,10 +90,37 @@ class Hash
       omit
     end
   end
+
+  private
+
+  unless {}.respond_to?(:_deep_transform_keys_in_object)
+    # support methods for deep transforming nested hashes and arrays
+    def _deep_transform_keys_in_object(object, &block)
+      case object
+      when Hash
+        object.each_with_object({}) do |(key, value), result|
+          result[yield(key)] = _deep_transform_keys_in_object(value, &block)
+        end
+      when Array
+        object.map { |element| _deep_transform_keys_in_object(element, &block) }
+      else
+        object
+      end
+    end
+  end
 end
 
+#
+# Monkey patches for the ruby Array
+#
 class Array
   unless [].respond_to?(:extract_options!)
+    #
+    # Extract the last argument if it is a hash
+    #
+    #
+    # @return [Hash]
+    #
     def extract_options!
       if last.is_a?(Hash) && last.instance_of?(Hash)
         pop

data/lib/sidekiq_unique_jobs/digests.rb

@@ -1,87 +1,65 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  # Utility module to help manage unique digests in redis.
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  module Digests
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Digests < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
     DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
     SCAN_PATTERN  = "*"
-    CHUNK_SIZE    = 100
-
-    include SidekiqUniqueJobs::Logging
-    include SidekiqUniqueJobs::Connection
-    extend self
 
-    # Return unique digests matching pattern
-    #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] count the maximum number to match
-    # @return [Array<String>] with unique digests
-    def all(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
-      redis { |conn| conn.sscan_each(UNIQUE_SET, match: pattern, count: count).to_a }
+    def initialize
+      super(DIGESTS)
     end
 
-    # Paginate unique digests
     #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] cursor the maximum number to match
-    # @param [Integer] page_size the current cursor position
+    # Adds a digest
     #
-    # @return [Array<String>] with unique digests
-    def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
-      redis do |conn|
-        total_size, digests = conn.multi do
-          conn.scard(UNIQUE_SET)
-          conn.sscan(UNIQUE_SET, cursor, match: pattern, count: page_size)
-        end
-
-        [total_size, digests[0], digests[1]]
-      end
-    end
-
-    # Get a total count of unique digests
+    # @param [String] digest the digest to add
     #
-    # @return [Integer] number of digests
-    def count
-      redis { |conn| conn.scard(UNIQUE_SET) }
+    def add(digest)
+      redis { |conn| conn.zadd(key, now_f, digest) }
     end
 
-    # Deletes unique digest either by a digest or pattern
+    # Deletes unique digests by pattern
     #
-    # @param [String] digest the full digest to delete
     # @param [String] pattern a key pattern to match with
     # @param [Integer] count the maximum number
-    # @raise [ArgumentError] when both pattern and digest are nil
     # @return [Array<String>] with unique digests
-    def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
-      warn("#{self}.#{__method__} has been deprecated and will be removed in a future version")
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = entries(pattern: pattern, count: count).keys
+        redis { |conn| BatchDelete.call(digests, conn) }
+      end
 
-      return delete_by_pattern(pattern, count: count) if pattern
-      return delete_by_digest(digest) if digest
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
 
-      raise ArgumentError, "either digest or pattern need to be provided"
+      result
     end
 
-    # Deletes unique digest either by a digest or pattern
+    # Delete unique digests by digest
+    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
     #
-    # @param [String] digest the full digest to delete
+    # @param [String] digest a unique digest to delete
     def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
       result, elapsed = timed do
-        Scripts.call(:delete_by_digest, nil, keys: [
-                       UNIQUE_SET,
-                       digest,
-                       "#{digest}:EXISTS",
-                       "#{digest}:GRABBED",
-                       "#{digest}:AVAILABLE",
-                       "#{digest}:VERSION",
-                       "#{digest}:RUN:EXISTS",
-                       "#{digest}:RUN:GRABBED",
-                       "#{digest}:RUN:AVAILABLE",
-                       "#{digest}:RUN:VERSION",
-                     ])
-
-        count
+        call_script(:delete_by_digest, [
+                      digest,
+                      "#{digest}:QUEUED",
+                      "#{digest}:PRIMED",
+                      "#{digest}:LOCKED",
+                      "#{digest}:RUN",
+                      "#{digest}:RUN:QUEUED",
+                      "#{digest}:RUN:PRIMED",
+                      "#{digest}:RUN:LOCKED",
+                      key,
+                    ])
       end
 
       log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
@@ -89,55 +67,48 @@ module SidekiqUniqueJobs
       result
     end
 
-    # Deletes unique digests by pattern
     #
-    # @param [String] pattern a key pattern to match with
-    # @param [Integer] count the maximum number
-    # @return [Array<String>] with unique digests
-    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
-      result, elapsed = timed do
-        digests = all(pattern: pattern, count: count)
-        batch_delete(digests)
-        digests.size
-      end
+    # The entries in this sorted set
+    #
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] count DEFAULT_COUNT the number of entries to return
+    #
+    # @return [Array<String>] an array of digests matching the given pattern
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      options = {}
+      options[:match] = pattern
+      options[:count] = count
 
-      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+      result = redis { |conn| conn.zscan_each(key, **options).to_a }
 
-      result
+      result.each_with_object({}) do |entry, hash|
+        hash[entry[0]] = entry[1]
+      end
     end
 
-    private
-
-    def batch_delete(digests) # rubocop:disable Metrics/MethodLength
+    #
+    # Returns a paginated
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] page_size 100 the size per page
+    #
+    # @return [Array<Integer, Integer, Array<Lock>>] total_size, next_cursor, locks
+    #
+    def page(cursor: 0, pattern: SCAN_PATTERN, page_size: 100)
       redis do |conn|
-        digests.each_slice(CHUNK_SIZE) do |chunk|
-          conn.pipelined do
-            chunk.each do |digest|
-              conn.del digest
-              conn.srem(UNIQUE_SET, digest)
-              conn.del("#{digest}:EXISTS")
-              conn.del("#{digest}:GRABBED")
-              conn.del("#{digest}:VERSION")
-              conn.del("#{digest}:AVAILABLE")
-              conn.del("#{digest}:RUN:EXISTS")
-              conn.del("#{digest}:RUN:GRABBED")
-              conn.del("#{digest}:RUN:VERSION")
-              conn.del("#{digest}:RUN:AVAILABLE")
-            end
-          end
+        total_size, digests = conn.multi do
+          conn.zcard(key)
+          conn.zscan(key, cursor, match: pattern, count: page_size)
         end
-      end
-    end
 
-    def timed
-      start = current_time
-      result = yield
-      elapsed = (current_time - start).round(2)
-      [result, elapsed]
-    end
-
-    def current_time
-      Time.now
+        [
+          total_size,
+          digests[0], # next_cursor
+          digests[1].map { |digest, score| Lock.new(digest, time: score) }, # entries
+        ]
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -1,29 +1,95 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
+  #
+  # Base class for all exceptions raised from the gem
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UniqueJobsError < ::RuntimeError
+  end
+
   # Error raised when a Lua script fails to execute
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class Conflict < StandardError
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Conflict < UniqueJobsError
     def initialize(item)
-      super("Item with the key: #{item[UNIQUE_DIGEST_KEY]} is already scheduled or processing")
+      super("Item with the key: #{item[LOCK_DIGEST]} is already scheduled or processing")
+    end
+  end
+
+  #
+  # Error raised when trying to add a duplicate lock
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateLock < UniqueJobsError
+  end
+
+  #
+  # Error raised when trying to add a duplicate stragegy
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateStrategy < UniqueJobsError
+  end
+
+  #
+  # Error raised when an invalid argument is given
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidArgument < UniqueJobsError
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidWorker < UniqueJobsError
+    def initialize(lock_config)
+      super(<<~FAILURE_MESSAGE)
+        Expected #{lock_config.worker} to have valid sidekiq options but found the following problems:
+        #{lock_config.errors_as_string}
+      FAILURE_MESSAGE
     end
   end
 
-  # Error raised from {OnConflict::Raise}
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class InvalidUniqueArguments < UniqueJobsError
+    def initialize(options)
+      given            = options[:given]
+      worker_class     = options[:worker_class]
+      lock_args_method = options[:lock_args_method]
+      lock_args_meth   = worker_class.method(lock_args_method)
+      num_args         = lock_args_meth.arity
+      source_location  = lock_args_meth.source_location
+
+      super(
+        "#{worker_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
+        "\n\n" \
+        "   #{source_location.join(':')}"
+      )
+    end
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class ScriptError < StandardError
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Redis::CommandError] source_exception exception to handle
-    def initialize(file_name:, source_exception:)
-      super("Problem compiling #{file_name}. Message: #{source_exception.message}")
+  class NotUniqueWorker < UniqueJobsError
+    def initialize(options)
+      super("#{options[:class]} is not configured for uniqueness. Missing the key `:lock` in #{options.inspect}")
     end
   end
 
   # Error raised from {OptionsWithFallback#lock_class}
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class UnknownLock < StandardError
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class UnknownLock < UniqueJobsError
   end
 end