sidekiq-unique-jobs 3.0.11 → 8.0.10

data/lib/sidekiq_unique_jobs/connection.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Shared module for dealing with redis connections
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Connection
+    def self.included(base)
+      base.send(:extend, self)
+    end
+
+    # Creates a connection to redis
+    # @return [Sidekiq::RedisConnection] a connection to redis
+    def redis(_r_pool = nil, &block)
+      Sidekiq.redis do |conn|
+        conn.with(&block)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+#
+# Module with constants to avoid string duplication
+#
+# @author Mikael Henriksson <mikael@mhenrixon.com>
+#
+module SidekiqUniqueJobs
+  ARGS                  = "args"
+  APARTMENT             = "apartment"
+  AT                    = "at"
+  CHANGELOGS            = "uniquejobs:changelog"
+  CLASS                 = "class"
+  CREATED_AT            = "created_at"
+  DEAD_VERSION          = "uniquejobs:dead"
+  DIGESTS               = "uniquejobs:digests"
+  EXPIRING_DIGESTS      = "uniquejobs:expiring_digests"
+  ORPHANED_DIGESTS      = "uniquejobs:orphaned_digests"
+  ERRORS                = "errors"
+  JID                   = "jid"
+  LIMIT                 = "limit"
+  LIVE_VERSION          = "uniquejobs:live"
+  LOCK                  = "lock"
+  LOCK_ARGS             = "lock_args"
+  LOCK_ARGS_METHOD      = "lock_args_method"
+  LOCK_DIGEST           = "lock_digest"
+  LOCK_EXPIRATION       = "lock_expiration"
+  LOCK_INFO             = "lock_info"
+  LOCK_LIMIT            = "lock_limit"
+  LOCK_PREFIX           = "lock_prefix"
+  LOCK_TIMEOUT          = "lock_timeout"
+  LOCK_TTL              = "lock_ttl"
+  LOCK_TYPE             = "lock_type"
+  ON_CLIENT_CONFLICT    = "on_client_conflict"
+  ON_CONFLICT           = "on_conflict"
+  ON_SERVER_CONFLICT    = "on_server_conflict"
+  PAYLOAD               = "payload"
+  PROCESSES             = "processes"
+  QUEUE                 = "queue"
+  RETRY                 = "retry"
+  SCHEDULE              = "schedule"
+  TIME                  = "time"
+  TIMEOUT               = "timeout"
+  TTL                   = "ttl"
+  TYPE                  = "type"
+  UNIQUE                = "unique"
+  UNIQUE_ACROSS_QUEUES  = "unique_across_queues"
+  UNIQUE_ACROSS_WORKERS = "unique_across_workers"
+  UNIQUE_ARGS           = "unique_args"
+  UNIQUE_ARGS_METHOD    = "unique_args_method"
+  UNIQUE_DIGEST         = "unique_digest"
+  UNIQUE_PREFIX         = "unique_prefix"
+  UNIQUE_REAPER         = "uniquejobs:reaper"
+  WORKER                = "worker"
+end

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+# :nocov:
+
+#
+# Monkey patches for the ruby Hash
+#
+class Hash
+  unless {}.respond_to?(:slice)
+    #
+    # Returns only the matching keys in a new hash
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
+    def slice(*keys)
+      keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
+      keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
+    end
+  end
+
+  unless {}.respond_to?(:deep_stringify_keys)
+    #
+    # Depp converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_stringify_keys
+      deep_transform_keys(&:to_s)
+    end
+  end
+
+  unless {}.respond_to?(:deep_transform_keys)
+    #
+    # Deep transfor all keys by yielding to the caller
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_transform_keys(&block)
+      _deep_transform_keys_in_object(self, &block)
+    end
+  end
+
+  unless {}.respond_to?(:stringify_keys)
+    #
+    # Converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def stringify_keys
+      transform_keys(&:to_s)
+    end
+  end
+
+  unless {}.respond_to?(:transform_keys)
+    #
+    # Transforms all keys by yielding to the caller
+    #
+    #
+    # @return [Hash]
+    #
+    def transform_keys
+      result = {}
+      each_key do |key|
+        result[yield(key)] = self[key]
+      end
+      result
+    end
+  end
+
+  unless {}.respond_to?(:slice!)
+    #
+    # Removes all keys not provided from the current hash and returns it
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
+    def slice!(*keys)
+      keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
+      omit = slice(*self.keys - keys)
+      hash = slice(*keys)
+      hash.default      = default
+      hash.default_proc = default_proc if default_proc
+      replace(hash)
+      omit
+    end
+  end
+
+  private
+
+  unless {}.respond_to?(:_deep_transform_keys_in_object)
+    # support methods for deep transforming nested hashes and arrays
+    def _deep_transform_keys_in_object(object, &block)
+      case object
+      when Hash
+        object.each_with_object(self.class.new) do |(key, value), result|
+          result[yield(key)] = _deep_transform_keys_in_object(value, &block)
+        end
+      when Array
+        object.map { |element| _deep_transform_keys_in_object(element, &block) }
+      else
+        object
+      end
+    end
+  end
+end
+
+#
+# Monkey patches for the ruby Array
+#
+class Array
+  unless [].respond_to?(:extract_options!)
+    #
+    # Extract the last argument if it is a hash
+    #
+    #
+    # @return [Hash]
+    #
+    def extract_options!
+      if last.is_a?(Hash) && last.instance_of?(Hash)
+        pop
+      else
+        {}
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/deprecation.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Deprecation provides logging of deprecations
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Deprecation
+    #
+    # Mute warnings from this gem in a threaded context
+    #
+    #
+    # @return [void] <description>
+    #
+    # @yieldreturn [void]
+    def self.muted
+      orig_val = Thread.current[:uniquejobs_mute_deprecations]
+      Thread.current[:uniquejobs_mute_deprecations] = true
+      yield
+    ensure
+      Thread.current[:uniquejobs_mute_deprecations] = orig_val
+    end
+
+    #
+    # Check if deprecation warnings have been muted
+    #
+    #
+    # @return [true,false]
+    #
+    def self.muted?
+      Thread.current[:uniquejobs_mute_deprecations] == true
+    end
+
+    #
+    # Warn about deprecation
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      warn "DEPRECATION WARNING: #{msg}"
+      nil
+    end
+
+    #
+    # Warn about deprecation and provide a context
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn_with_backtrace(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      trace = "\n\nCALLED FROM:\n#{caller.join("\n")}"
+      warn(msg + trace)
+
+      nil
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/digests.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Digests < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
+    DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
+    SCAN_PATTERN  = "*"
+    #
+    # @return [Array(String, String, String, String)] The empty runtime or queuetime keys.
+    EMPTY_KEYS_SEGMENT = ["", "", "", ""].freeze
+
+    def initialize(digests_key = DIGESTS)
+      super(digests_key)
+    end
+
+    #
+    # Adds a digest
+    #
+    # @param [String] digest the digest to add
+    #
+    def add(digest)
+      redis { |conn| conn.zadd(key, now_f, digest) }
+    end
+
+    # Deletes unique digests by pattern
+    #
+    # @param [String] pattern a key pattern to match with
+    # @param [Integer] count the maximum number
+    # @return [Hash<String,Float>] Hash mapping of digest matching the given pattern and score
+
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = entries(pattern: pattern, count: count).keys
+        redis { |conn| BatchDelete.call(digests, conn) }
+      end
+
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    # Delete unique digests by digest
+    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
+    #
+    # @param [String] digest a unique digest to delete
+    # @param queuetime [Boolean] Whether to delete queue locks.
+    # @param runtime [Boolean] Whether to delete run locks.
+    def delete_by_digest(digest, queuetime: true, runtime: true)
+      result, elapsed = timed do
+        call_script(
+          :delete_by_digest,
+          queuetime_keys(queuetime ? digest : nil) + runtime_keys(runtime ? digest : nil) + [key],
+        )
+      end
+
+      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    #
+    # The entries in this sorted set
+    #
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] count DEFAULT_COUNT the number of entries to return
+    #
+    # @return [Array<String>] an array of digests matching the given pattern
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      redis { |conn| conn.zscan(key, match: pattern, count: count).to_a }.to_h
+    end
+
+    #
+    # Returns a paginated
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] page_size 100 the size per page
+    #
+    # @return [Array<Integer, Integer, Array<Lock>>] total_size, next_cursor, locks
+    #
+    def page(cursor: 0, pattern: SCAN_PATTERN, page_size: 100)
+      redis do |conn|
+        total_size, digests = conn.multi do |pipeline|
+          pipeline.zcard(key)
+          pipeline.zscan(key, cursor, match: pattern, count: page_size)
+        end
+
+        # NOTE: When debugging, check the last item in the returned array.
+        [
+          total_size.to_i,
+          digests[0].to_i, # next_cursor
+          digests[1].each_slice(2).map { |digest, score| Lock.new(digest, time: score) }, # entries
+        ]
+      end
+    end
+
+    private
+
+    # @param digest [String, nil] The digest to form runtime keys from.
+    # @return [Array(String, String, String, String)] The list of runtime keys or empty strings if +digest+ was +nil+.
+    def runtime_keys(digest)
+      return EMPTY_KEYS_SEGMENT unless digest
+
+      [
+        "#{digest}:RUN",
+        "#{digest}:RUN:QUEUED",
+        "#{digest}:RUN:PRIMED",
+        "#{digest}:RUN:LOCKED",
+      ]
+    end
+
+    # @param digest [String, nil] The digest to form queuetime keys from.
+    # @return [Array(String, String, String, String)] The list of queuetime keys or empty strings if +digest+ was +nil+.
+    def queuetime_keys(digest)
+      return EMPTY_KEYS_SEGMENT unless digest
+
+      [
+        digest,
+        "#{digest}:QUEUED",
+        "#{digest}:PRIMED",
+        "#{digest}:LOCKED",
+      ]
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Base class for all exceptions raised from the gem
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UniqueJobsError < ::RuntimeError
+  end
+
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Conflict < UniqueJobsError
+    def initialize(item)
+      super("Item with the key: #{item[LOCK_DIGEST]} is already scheduled or processing")
+    end
+  end
+
+  #
+  # Raised when no block was given
+  #
+  class NoBlockGiven < SidekiqUniqueJobs::UniqueJobsError; end
+
+  #
+  # Raised when a notification has been mistyped
+  #
+  class NoSuchNotificationError < UniqueJobsError; end
+
+  #
+  # Error raised when trying to add a duplicate lock
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateLock < UniqueJobsError
+  end
+
+  #
+  # Error raised when trying to add a duplicate stragegy
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateStrategy < UniqueJobsError
+  end
+
+  #
+  # Error raised when an invalid argument is given
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidArgument < UniqueJobsError
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidWorker < UniqueJobsError
+    def initialize(lock_config)
+      super(<<~FAILURE_MESSAGE)
+        Expected #{lock_config.worker} to have valid sidekiq options but found the following problems:
+        #{lock_config.errors_as_string}
+      FAILURE_MESSAGE
+    end
+  end
+
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class InvalidUniqueArguments < UniqueJobsError
+    def initialize(options)
+      given            = options[:given]
+      job_class        = options[:job_class]
+      lock_args_method = options[:lock_args_method]
+      lock_args_meth   = job_class.method(lock_args_method)
+      num_args         = lock_args_meth.arity
+      source_location  = lock_args_meth.source_location
+
+      super(
+        "#{job_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
+        "\n\n" \
+        "   #{source_location.join(':')}"
+      )
+    end
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class NotUniqueWorker < UniqueJobsError
+    def initialize(options)
+      super("#{options[:class]} is not configured for uniqueness. Missing the key `:lock` in #{options.inspect}")
+    end
+  end
+
+  # Error raised from {OptionsWithFallback#lock_class}
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class UnknownLock < UniqueJobsError
+  end
+end

data/lib/sidekiq_unique_jobs/expiring_digests.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class ExpiringDigests provides access to the expiring digests used by until_expired locks
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class ExpiringDigests < Digests
+    def initialize
+      super(EXPIRING_DIGESTS)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/job.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Utility class to append uniqueness to the sidekiq job hash
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Job
+    extend self
+
+    # Adds lock, timeout, expiration, lock_args, lock_prefix, and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def prepare(item)
+      stringify_on_conflict_hash(item)
+      add_lock_type(item)
+      add_lock_timeout(item)
+      add_lock_ttl(item)
+      add_digest(item)
+    end
+
+    # Adds lock_args, lock_prefix and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def add_digest(item)
+      add_lock_prefix(item)
+      add_lock_args(item)
+      add_lock_digest(item)
+
+      item
+    end
+
+    private
+
+    def stringify_on_conflict_hash(item)
+      on_conflict = item[ON_CONFLICT]
+      return unless on_conflict.is_a?(Hash)
+
+      item[ON_CONFLICT] = on_conflict.deep_stringify_keys
+    end
+
+    def add_lock_ttl(item)
+      item[LOCK_TTL] = SidekiqUniqueJobs::LockTTL.calculate(item)
+    end
+
+    def add_lock_timeout(item)
+      item[LOCK_TIMEOUT] = SidekiqUniqueJobs::LockTimeout.calculate(item)
+    end
+
+    def add_lock_args(item)
+      item[LOCK_ARGS] = SidekiqUniqueJobs::LockArgs.call(item)
+    end
+
+    def add_lock_digest(item)
+      item[LOCK_DIGEST] = SidekiqUniqueJobs::LockDigest.call(item)
+    end
+
+    def add_lock_prefix(item)
+      item[LOCK_PREFIX] = SidekiqUniqueJobs.config.lock_prefix
+    end
+
+    def add_lock_type(item)
+      item[LOCK] ||= SidekiqUniqueJobs::LockType.call(item)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/json.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles loading and dumping of json
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module JSON
+    module_function
+
+    #
+    # Parses a JSON string into an object
+    #
+    # @param [String] string the object to parse
+    #
+    # @return [Object]
+    #
+    def load_json(string)
+      return if string.nil? || string.empty?
+
+      ::JSON.parse(string)
+    end
+
+    #
+    # Prevents trying JSON.load from raising errors given argument is a hash
+    #
+    # @param [String, Hash] string the JSON string to parse
+    #
+    # @return [Hash,Array]
+    #
+    def safe_load_json(string)
+      return string if string.is_a?(Hash)
+
+      load_json(string)
+    end
+
+    #
+    # Dumps an object into a JSON string
+    #
+    # @param [Object] object a JSON convertible object
+    #
+    # @return [String] a JSON string
+    #
+    def dump_json(object)
+      ::JSON.generate(object)
+    end
+  end
+end