sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

data/lib/sidekiq_unique_jobs/connection.rb

@@ -11,11 +11,12 @@ module SidekiqUniqueJobs
 
     # Creates a connection to redis
     # @return [Sidekiq::RedisConnection, ConnectionPool] a connection to redis
-    def redis(redis_pool = nil)
-      if redis_pool
-        redis_pool.with { |conn| yield conn }
+    def redis(r_pool = nil, &block)
+      r_pool ||= defined?(redis_pool) ? redis_pool : r_pool
+      if r_pool
+        r_pool.with(&block)
       else
-        Sidekiq.redis { |conn| yield conn }
+        Sidekiq.redis(&block)
       end
     end
   end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -6,28 +6,40 @@
 # @author Mikael Henriksson <mikael@zoolutions.se>
 #
 module SidekiqUniqueJobs
-  ARGS_KEY                  ||= "args"
-  APARTMENT                 ||= "apartment"
-  AT_KEY                    ||= "at"
-  CLASS_KEY                 ||= "class"
-  JAVA                      ||= "java"
-  JID_KEY                   ||= "jid"
-  LOCK_DIGEST_KEY           ||= "lock_digest"
-  LOCK_EXPIRATION_KEY       ||= "lock_expiration"
-  LOCK_TIMEOUT_KEY          ||= "lock_timeout"
-  LOCK_TTL_KEY              ||= "lock_ttl"
-  LOG_DUPLICATE_KEY         ||= "log_duplicate_payload"
-  QUEUE_KEY                 ||= "queue"
-  UNIQUE_ACROSS_QUEUES_KEY  ||= "unique_across_queues"
-  UNIQUE_ACROSS_WORKERS_KEY ||= "unique_across_workers"
-  UNIQUE_ARGS_KEY           ||= "unique_args"
-  UNIQUE_DIGEST_KEY         ||= "unique_digest"
-  UNIQUE_KEY                ||= "unique"
-  UNIQUE_SET                ||= "unique:keys"
-  LOCK_KEY                  ||= "lock"
-  ON_CONFLICT_KEY           ||= "on_conflict"
-  UNIQUE_ON_ALL_QUEUES_KEY  ||= "unique_on_all_queues" # TODO: Remove in v6.1
-  UNIQUE_PREFIX_KEY         ||= "unique_prefix"
-  RETRY_SET                 ||= "retry"
-  SCHEDULE_SET              ||= "schedule"
+  ARGS                  ||= "args"
+  AT                    ||= "at"
+  CHANGELOGS            ||= "uniquejobs:changelog"
+  CLASS                 ||= "class"
+  DEAD_VERSION          ||= "uniquejobs:dead"
+  DIGESTS               ||= "uniquejobs:digests"
+  ERRORS                ||= "errors"
+  JID                   ||= "jid"
+  LIMIT                 ||= "limit"
+  LIVE_VERSION          ||= "uniquejobs:live"
+  LOCK                  ||= "lock"
+  LOCK_EXPIRATION       ||= "lock_expiration"
+  LOCK_INFO             ||= "lock_info"
+  LOCK_LIMIT            ||= "lock_limit"
+  LOCK_PREFIX           ||= "lock_prefix"
+  LOCK_TIMEOUT          ||= "lock_timeout"
+  LOCK_TTL              ||= "lock_ttl"
+  LOCK_TYPE             ||= "lock_type"
+  LOG_DUPLICATE         ||= "log_duplicate"
+  ON_CLIENT_CONFLICT    ||= "on_client_conflict"
+  ON_CONFLICT           ||= "on_conflict"
+  ON_SERVER_CONFLICT    ||= "on_server_conflict"
+  QUEUE                 ||= "queue"
+  RETRY                 ||= "retry"
+  SCHEDULE              ||= "schedule"
+  TIME                  ||= "time"
+  TIMEOUT               ||= "timeout"
+  TTL                   ||= "ttl"
+  TYPE                  ||= "type"
+  UNIQUE                ||= "unique"
+  UNIQUE_ACROSS_QUEUES  ||= "unique_across_queues"
+  UNIQUE_ACROSS_WORKERS ||= "unique_across_workers"
+  UNIQUE_ARGS           ||= "unique_args"
+  UNIQUE_DIGEST         ||= "unique_digest"
+  UNIQUE_PREFIX         ||= "unique_prefix"
+  WORKER                ||= "worker"
 end

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -2,8 +2,18 @@
 
 # :nocov:
 
+#
+# Monkey patches for the ruby Hash
+#
 class Hash
   unless {}.respond_to?(:slice)
+    #
+    # Returns only the matching keys in a new hash
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
@@ -11,12 +21,24 @@ class Hash
   end
 
   unless {}.respond_to?(:stringify_keys)
+    #
+    # Converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
     def stringify_keys
       transform_keys(&:to_s)
     end
   end
 
   unless {}.respond_to?(:transform_keys)
+    #
+    # Transforms all keys by yielding to the caller
+    #
+    #
+    # @return [Hash]
+    #
     def transform_keys
       result = {}
       each_key do |key|
@@ -27,6 +49,13 @@ class Hash
   end
 
   unless {}.respond_to?(:slice!)
+    #
+    # Removes all keys not provided from the current hash and returns it
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice!(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       omit = slice(*self.keys - keys)
@@ -39,8 +68,17 @@ class Hash
   end
 end
 
+#
+# Monkey patches for the ruby Array
+#
 class Array
   unless [].respond_to?(:extract_options!)
+    #
+    # Extract the last argument if it is a hash
+    #
+    #
+    # @return [Hash]
+    #
     def extract_options!
       if last.is_a?(Hash) && last.instance_of?(Hash)
         pop

data/lib/sidekiq_unique_jobs/digests.rb

@@ -1,94 +1,100 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  # Utility module to help manage unique digests in redis.
+  #
+  # Class Changelogs provides access to the changelog entries
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
-  module Digests
+  #
+  class Digests < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
     DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
     SCAN_PATTERN  = "*"
-    CHUNK_SIZE    = 100
 
-    include SidekiqUniqueJobs::Logging
-    include SidekiqUniqueJobs::Connection
-    extend self
-
-    # Return unique digests matching pattern
-    #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] count the maximum number to match
-    # @return [Array<String>] with unique digests
-    def all(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
-      redis { |conn| conn.sscan_each(UNIQUE_SET, match: pattern, count: count).to_a }
+    def initialize
+      super(DIGESTS)
     end
 
-    # Paginate unique digests
     #
-    # @param [String] pattern a pattern to match with
-    # @param [Integer] cursor the maximum number to match
-    # @param [Integer] page_size the current cursor position
+    # Adds a digest
     #
-    # @return [Array<String>] with unique digests
-    def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
-      redis do |conn|
-        total_size, digests = conn.multi do
-          conn.scard(UNIQUE_SET)
-          conn.sscan(UNIQUE_SET, cursor, match: pattern, count: page_size)
-        end
-
-        [total_size, digests[0], digests[1]]
-      end
-    end
-
-    # Get a total count of unique digests
+    # @param [String] digest the digest to add
     #
-    # @return [Integer] number of digests
-    def count
-      redis { |conn| conn.scard(UNIQUE_SET) }
+    def add(digest)
+      redis { |conn| conn.zadd(key, now_f, digest) }
     end
 
+    #
     # Deletes unique digest either by a digest or pattern
     #
-    # @param [String] digest the full digest to delete
-    # @param [String] pattern a key pattern to match with
-    # @param [Integer] count the maximum number
-    # @raise [ArgumentError] when both pattern and digest are nil
+    # @overload call_script(digest: "abcdefab")
+    #   Call script with digest
+    #   @param [String] digest: a digest to delete
+    # @overload call_script(pattern: "*", count: 1_000)
+    #   Call script with pattern
+    #   @param [String] pattern: "*" a pattern to match
+    #   @param [String] count: DEFAULT_COUNT the number of keys to delete
+    #
+    # @raise [ArgumentError] when given neither pattern nor digest
+    #
     # @return [Array<String>] with unique digests
+    #
     def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
-      warn("#{self}.#{__method__} has been deprecated and will be removed in a future version")
-
       return delete_by_pattern(pattern, count: count) if pattern
       return delete_by_digest(digest) if digest
 
-      raise ArgumentError, "either digest or pattern need to be provided"
+      raise ArgumentError, "##{__method__} requires either a :digest or a :pattern"
     end
 
-    # Deletes unique digest either by a digest or pattern
     #
-    # @param [String] digest the full digest to delete
-    def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
-      result, elapsed = timed do
-        Scripts.call(:delete_by_digest, nil, keys: [
-                       UNIQUE_SET,
-                       digest,
-                       "#{digest}:EXISTS",
-                       "#{digest}:GRABBED",
-                       "#{digest}:AVAILABLE",
-                       "#{digest}:VERSION",
-                       "#{digest}:RUN:EXISTS",
-                       "#{digest}:RUN:GRABBED",
-                       "#{digest}:RUN:AVAILABLE",
-                       "#{digest}:RUN:VERSION",
-                     ])
-
-        count
+    # The entries in this sorted set
+    #
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] count DEFAULT_COUNT the number of entries to return
+    #
+    # @return [Array<String>] an array of digests matching the given pattern
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      options = {}
+      options[:match] = pattern
+      options[:count] = count if count
+
+      result = redis { |conn| conn.zscan_each(key, options).to_a }
+
+      result.each_with_object({}) do |entry, hash|
+        hash[entry[0]] = entry[1]
       end
+    end
 
-      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
+    #
+    # Returns a paginated
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] page_size 100 the size per page
+    #
+    # @return [Array<Integer, Integer, Array<Lock>>] total_size, next_cursor, locks
+    #
+    def page(cursor: 0, pattern: SCAN_PATTERN, page_size: 100)
+      redis do |conn|
+        total_size, digests = conn.multi do
+          conn.zcard(key)
+          conn.zscan(key, cursor, match: pattern, count: page_size)
+        end
 
-      result
+        [
+          total_size,
+          digests[0], # next_cursor
+          digests[1].map { |digest, score| Lock.new(digest, time: score) }, # entries
+        ]
+      end
     end
 
+    private
+
     # Deletes unique digests by pattern
     #
     # @param [String] pattern a key pattern to match with
@@ -96,9 +102,8 @@ module SidekiqUniqueJobs
     # @return [Array<String>] with unique digests
     def delete_by_pattern(pattern, count: DEFAULT_COUNT)
       result, elapsed = timed do
-        digests = all(pattern: pattern, count: count)
-        batch_delete(digests)
-        digests.size
+        digests = entries(pattern: pattern, count: count).keys
+        redis { |conn| BatchDelete.call(digests, conn) }
       end
 
       log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
@@ -106,38 +111,18 @@ module SidekiqUniqueJobs
       result
     end
 
-    private
-
-    def batch_delete(digests) # rubocop:disable Metrics/MethodLength
-      redis do |conn|
-        digests.each_slice(CHUNK_SIZE) do |chunk|
-          conn.pipelined do
-            chunk.each do |digest|
-              conn.del digest
-              conn.srem(UNIQUE_SET, digest)
-              conn.del("#{digest}:EXISTS")
-              conn.del("#{digest}:GRABBED")
-              conn.del("#{digest}:VERSION")
-              conn.del("#{digest}:AVAILABLE")
-              conn.del("#{digest}:RUN:EXISTS")
-              conn.del("#{digest}:RUN:GRABBED")
-              conn.del("#{digest}:RUN:VERSION")
-              conn.del("#{digest}:RUN:AVAILABLE")
-            end
-          end
-        end
+    # Delete unique digests by digest
+    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
+    #
+    # @param [String] digest a unique digest to delete
+    def delete_by_digest(digest)
+      result, elapsed = timed do
+        call_script(:delete_by_digest, [digest, key])
       end
-    end
 
-    def timed
-      start = current_time
-      result = yield
-      elapsed = (current_time - start).round(2)
-      [result, elapsed]
-    end
+      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
 
-    def current_time
-      Time.now
+      result
     end
   end
 end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -1,29 +1,173 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
+  #
+  # Base class for all exceptions raised from the gem
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class UniqueJobsError < ::RuntimeError
+  end
+
   # Error raised when a Lua script fails to execute
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
-  class Conflict < StandardError
+  class Conflict < UniqueJobsError
     def initialize(item)
-      super("Item with the key: #{item[UNIQUE_DIGEST_KEY]} is already scheduled or processing")
+      super("Item with the key: #{item[UNIQUE_DIGEST]} is already scheduled or processing")
+    end
+  end
+
+  #
+  # Error raised when trying to add a duplicate lock
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class DuplicateLock < UniqueJobsError
+  end
+
+  #
+  # Error raised when trying to add a duplicate stragegy
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class DuplicateStrategy < UniqueJobsError
+  end
+
+  #
+  # Error raised when an invalid argument is given
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class InvalidArgument < UniqueJobsError
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class InvalidWorker < UniqueJobsError
+    def initialize(lock_config)
+      super(<<~FAILURE_MESSAGE)
+        Expected #{lock_config.worker} to have valid sidekiq options but found the following problems:
+        #{lock_config.errors_as_string}
+      FAILURE_MESSAGE
+    end
+  end
+
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  class InvalidUniqueArguments < UniqueJobsError
+    def initialize(given:, worker_class:, unique_args_method:)
+      uniq_args_meth  = worker_class.method(unique_args_method)
+      num_args        = uniq_args_meth.arity
+      # source_location = uniq_args_meth.source_location
+
+      super(
+        "#{worker_class}#unique_args takes #{num_args} arguments, received #{given.inspect}"
+      )
+    end
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class NotUniqueWorker < UniqueJobsError
+    def initialize(options: {})
+      super("#{options[:class]} is not configured for uniqueness. Missing the key `:lock` in #{options.inspect}")
     end
   end
 
   # Error raised from {OnConflict::Raise}
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
-  class ScriptError < StandardError
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Redis::CommandError] source_exception exception to handle
-    def initialize(file_name:, source_exception:)
-      super("Problem compiling #{file_name}. Message: #{source_exception.message}")
+  class ScriptError < UniqueJobsError
+    # Reformats errors raised by redis representing failures while executing
+    # a lua script. The default errors have confusing messages and backtraces,
+    # and a type of +RuntimeError+. This class improves the message and
+    # modifies the backtrace to include the lua script itself in a reasonable
+    # way.
+
+    PATTERN  = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/.freeze
+    LIB_PATH = File.expand_path("..", __dir__)
+    CONTEXT_LINE_NUMBER = 3
+
+    attr_reader :error, :file, :content
+
+    # Is this error one that should be reformatted?
+    #
+    # @param error [StandardError] the original error raised by redis
+    # @return [Boolean] is this an error that should be reformatted?
+    def self.intercepts?(error)
+      error.message =~ PATTERN
+    end
+
+    # Initialize a new {ScriptError} from an existing redis error, adjusting
+    # the message and backtrace in the process.
+    #
+    # @param error [StandardError] the original error raised by redis
+    # @param file [Pathname] full path to the lua file the error ocurred in
+    # @param content [String] lua file content the error ocurred in
+    # :nocov:
+    def initialize(error, file, content)
+      @error        = error
+      @file         = file
+      @content      = content
+      @backtrace    = @error.backtrace
+
+      @error.message.match(PATTERN) do |regexp_match|
+        line_number   = regexp_match[2].to_i
+        message       = regexp_match[3]
+        error_context = generate_error_context(content, line_number)
+
+        super("#{message}\n\n#{error_context}\n\n")
+        set_backtrace(generate_backtrace(file, line_number))
+      end
+    end
+
+    private
+
+    # :nocov:
+    def generate_error_context(content, line_number)
+      lines                 = content.lines.to_a
+      beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+      ending_line_number    = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+      line_number_width     = ending_line_number.to_s.length
+
+      (beginning_line_number..ending_line_number).map do |number|
+        indicator = (number == line_number) ? "=>" : "  "
+        formatted_number = format("%#{line_number_width}d", number)
+        " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+      end.join.chomp
+    end
+
+    # :nocov:
+    def generate_backtrace(file, line_number)
+      pre_gem                         = backtrace_before_entering_gem(@backtrace)
+      index_of_first_unique_jobs_line = (@backtrace.size - pre_gem.size - 1)
+      pre_gem.unshift(@backtrace[index_of_first_unique_jobs_line])
+      pre_gem.unshift("#{file}:#{line_number}")
+      pre_gem
+    end
+
+    # :nocov:
+    def backtrace_before_entering_gem(backtrace)
+      backtrace.reverse.take_while { |line| !line_from_gem(line) }.reverse
+    end
+
+    # :nocov:
+    def line_from_gem(line)
+      line.split(":").first.include?(LIB_PATH)
     end
   end
 
   # Error raised from {OptionsWithFallback#lock_class}
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
-  class UnknownLock < StandardError
+  class UnknownLock < UniqueJobsError
   end
 end