sidekiq-unique-jobs 4.0.18 → 7.1.28

data/lib/sidekiq_unique_jobs/config.rb

@@ -1,23 +1,319 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  class Config < OpenStruct
-    CONFIG_ACCESSORS = [
-      :unique_prefix,
-      :default_queue_lock_expiration,
-      :default_run_lock_expiration,
-      :default_lock,
-      :redis_mode
-    ].freeze
+  # ThreadSafe config exists to be able to document the config class without errors
+  ThreadSafeConfig = Concurrent::MutableStruct.new("ThreadSafeConfig",
+                                                   :lock_timeout,
+                                                   :lock_ttl,
+                                                   :enabled,
+                                                   :lock_prefix,
+                                                   :logger,
+                                                   :logger_enabled,
+                                                   :locks,
+                                                   :strategies,
+                                                   :debug_lua,
+                                                   :max_history,
+                                                   :reaper,
+                                                   :reaper_count,
+                                                   :reaper_interval,
+                                                   :reaper_timeout,
+                                                   :reaper_resurrector_interval,
+                                                   :reaper_resurrector_enabled,
+                                                   :lock_info,
+                                                   :raise_on_config_error,
+                                                   :current_redis_version)
+
+  #
+  # Shared class for dealing with gem configuration
+  #
+  # @author Mauro Berlanda <mauro.berlanda@gmail.com>
+  # rubocop:disable Metrics/ClassLength
+  class Config < ThreadSafeConfig
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available queued locks
+    LOCKS_WHILE_ENQUEUED = {
+      until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+      while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available fulltime locks
+    LOCKS_FROM_PUSH_TO_PROCESSED = {
+      until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
+      until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available locks without unlock
+    LOCKS_WITHOUT_UNLOCK = {
+      until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available runtime/client locks
+    LOCKS_WHEN_BUSY = {
+      around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available default locks
+    LOCKS =
+      LOCKS_WHEN_BUSY.dup
+                     .merge(LOCKS_WHILE_ENQUEUED.dup)
+                     .merge(LOCKS_WITHOUT_UNLOCK.dup)
+                     .merge(LOCKS_FROM_PUSH_TO_PROCESSED.dup)
+                     .freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::OnConflict::Strategy] all available default strategies
+    STRATEGIES = {
+      log: SidekiqUniqueJobs::OnConflict::Log,
+      raise: SidekiqUniqueJobs::OnConflict::Raise,
+      reject: SidekiqUniqueJobs::OnConflict::Reject,
+      replace: SidekiqUniqueJobs::OnConflict::Replace,
+      reschedule: SidekiqUniqueJobs::OnConflict::Reschedule,
+    }.freeze
+
+    #
+    # @return ['uniquejobs'] by default we use this prefix
+    PREFIX                = "uniquejobs"
+    #
+    # @return [0] by default don't wait for locks
+    LOCK_TIMEOUT          = 0
+    #
+    # @return [nil]
+    LOCK_TTL              = nil
+    #
+    # @return [true,false] by default false (don't disable logger)
+    LOGGER_ENABLED        = true
+    #
+    # @return [true] by default the gem is enabled
+    ENABLED               = true
+    #
+    # @return [false] by default we don't debug the lua scripts because it is slow
+    DEBUG_LUA             = false
+    #
+    # @return [1_000] use a changelog history of 1_000 entries by default
+    MAX_HISTORY           = 1_000
+    #
+    # @return [:ruby] prefer the ruby reaper by default since the lua reaper still has problems
+    REAPER                = :ruby
+    #
+    # @return [1_000] reap 1_000 orphaned locks at a time by default
+    REAPER_COUNT          = 1_000
+    #
+    # @return [600] reap locks every 10 minutes
+    REAPER_INTERVAL       = 600
+    #
+    # @return [10] stop reaper after 10 seconds
+    REAPER_TIMEOUT        = 10
+    #
+    # @return [3600] check if reaper is dead each 3600 seconds
+    REAPER_RESURRECTOR_INTERVAL = 3600
+
+    #
+    # @return [false] enable reaper resurrector
+    REAPER_RESURRECTOR_ENABLED = false
+
+    #
+    # @return [false] while useful it also adds overhead so disable lock_info by default
+    USE_LOCK_INFO         = false
+    #
+    # @return [false] by default we don't raise validation errors for workers
+    RAISE_ON_CONFIG_ERROR = false
+    #
+    # @return [0.0.0] default redis version is only to avoid NoMethodError on nil
+    REDIS_VERSION         = "0.0.0"
+
+    #
+    # Returns a default configuration
+    #
+    # @example
+    #   SidekiqUniqueJobs::Config.default => <concurrent/mutable_struct/thread_safe_config SidekiqUniqueJobs::Config {
+    #   default_lock_timeout: 0,
+    #   default_lock_ttl: nil,
+    #   enabled: true,
+    #   lock_prefix: "uniquejobs",
+    #   logger: #<Sidekiq::Logger:0x00007f81e096b0e0 @level=1 ...>,
+    #   locks: {
+    #     around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
+    #     until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+    #     while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
+    #     until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+    #     until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
+    #     until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted
+    #   },
+    #   strategies: {
+    #     log: SidekiqUniqueJobs::OnConflict::Log,
+    #     raise: SidekiqUniqueJobs::OnConflict::Raise,
+    #     reject: SidekiqUniqueJobs::OnConflict::Reject,
+    #     replace: SidekiqUniqueJobs::OnConflict::Replace,
+    #     reschedule: SidekiqUniqueJobs::OnConflict::Reschedule
+    #   },
+    #   debug_lua: false,
+    #   max_history: 1000,
+    #   reaper:: ruby,
+    #   reaper_count: 1000,
+    #   lock_info: false,
+    #   raise_on_config_error: false,
+    #   }>
+    #
+    #
+    # @return [SidekiqUniqueJobs::Config] a default configuration
+    #
+    def self.default # rubocop:disable Metrics/MethodLength
+      new(
+        LOCK_TIMEOUT,
+        LOCK_TTL,
+        ENABLED,
+        PREFIX,
+        Sidekiq.logger,
+        LOGGER_ENABLED,
+        LOCKS,
+        STRATEGIES,
+        DEBUG_LUA,
+        MAX_HISTORY,
+        REAPER,
+        REAPER_COUNT,
+        REAPER_INTERVAL,
+        REAPER_TIMEOUT,
+        REAPER_RESURRECTOR_INTERVAL,
+        REAPER_RESURRECTOR_ENABLED,
+        USE_LOCK_INFO,
+        RAISE_ON_CONFIG_ERROR,
+        REDIS_VERSION,
+      )
+    end
 
-    def inline_testing_enabled?
-      testing_enabled? && Sidekiq::Testing.inline?
+    #
+    # Set the default_lock_ttl
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [<type>] <description>
+    #
+    def default_lock_ttl=(obj)
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl=` instead."
+      self.lock_ttl = obj
     end
 
-    def mocking?
-      inline_testing_enabled? && redis_test_mode.to_sym == :mock
+    #
+    # Set new value for default_lock_timeout
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [Integer]
+    #
+    def default_lock_timeout=(obj)
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout=` instead."
+      self.lock_timeout = obj
+    end
+
+    #
+    # Default lock TTL (Time To Live)
+    # @deprecated
+    #
+    # @return [nil, Integer] configured value or nil
+    #
+    def default_lock_ttl
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl` instead."
+      lock_ttl
+    end
+
+    #
+    # Default Lock Timeout
+    # @deprecated
+    #
+    #
+    # @return [nil, Integer] configured value or nil
+    #
+    def default_lock_timeout
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout` instead."
+      lock_timeout
+    end
+
+    #
+    # Memoized variable to get the class name
+    #
+    #
+    # @return [String] name of the class
+    #
+    def class_name
+      @class_name ||= self.class.name
+    end
+
+    #
+    # Adds a lock type to the configuration. It will raise if the lock exists already
+    #
+    # @example Add a custom lock
+    #   add_lock(:my_lock, CustomLocks::MyLock)
+    #
+    # @raise DuplicateLock when the name already exists
+    #
+    # @param [String, Symbol] name the name of the lock
+    # @param [Class] klass the class describing the lock
+    #
+    # @return [void]
+    #
+    def add_lock(name, klass)
+      lock_sym = name.to_sym
+      raise DuplicateLock, ":#{name} already defined, please use another name" if locks.key?(lock_sym)
+
+      new_locks = locks.dup.merge(lock_sym => klass).freeze
+      self.locks = new_locks
+    end
+
+    #
+    # Adds an on_conflict strategy to the configuration.
+    #
+    # @example Add a custom strategy
+    #   add_lock(:my_strategy, CustomStrategies::MyStrategy)
+    #
+    # @raise [DuplicateStrategy] when the name already exists
+    #
+    # @param [String] name the name of the custom strategy
+    # @param [Class] klass the class describing the strategy
+    #
+    def add_strategy(name, klass)
+      strategy_sym = name.to_sym
+      raise DuplicateStrategy, ":#{name} already defined, please use another name" if strategies.key?(strategy_sym)
+
+      new_strategies = strategies.dup.merge(strategy_sym => klass).freeze
+      self.strategies = new_strategies
     end
 
-    def testing_enabled?
-      Sidekiq.const_defined?(TESTING_CONSTANT, false) && Sidekiq::Testing.enabled?
+    #
+    # The current version of redis
+    #
+    #
+    # @return [String] a version string eg. `5.0.1`
+    #
+    def redis_version
+      self.current_redis_version = SidekiqUniqueJobs.fetch_redis_version if current_redis_version == REDIS_VERSION
+      current_redis_version
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/sidekiq_unique_jobs/connection.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Shared module for dealing with redis connections
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Connection
+    def self.included(base)
+      base.send(:extend, self)
+    end
+
+    # Creates a connection to redis
+    # @return [Sidekiq::RedisConnection, ConnectionPool] a connection to redis
+    def redis(r_pool = nil, &block)
+      r_pool ||= defined?(redis_pool) ? redis_pool : r_pool
+      if r_pool
+        r_pool.with(&block)
+      else
+        Sidekiq.redis(&block)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -1,18 +1,54 @@
+# frozen_string_literal: true
+
+#
+# Module with constants to avoid string duplication
+#
+# @author Mikael Henriksson <mikael@mhenrixon.com>
+#
 module SidekiqUniqueJobs
-  ARGS_KEY ||= 'args'.freeze
-  AT_KEY ||= 'at'.freeze
-  CLASS_KEY ||= 'class'.freeze
-  JID_KEY ||= 'jid'.freeze
-  LOG_DUPLICATE_KEY ||= 'log_duplicate_payload'.freeze
-  QUEUE_KEY ||= 'queue'.freeze
-  QUEUE_LOCK_TIMEOUT_KEY ||= 'unique_expiration'.freeze
-  RUN_LOCK_TIMEOUT_KEY ||= 'run_lock_expiration'.freeze
-  TESTING_CONSTANT ||= 'Testing'.freeze
-  UNIQUE_KEY ||= 'unique'.freeze
-  UNIQUE_LOCK_KEY ||= 'unique_lock'.freeze
-  UNIQUE_ARGS_KEY ||= 'unique_args'.freeze
-  UNIQUE_PREFIX_KEY ||= 'unique_prefix'.freeze
-  UNIQUE_DIGEST_KEY ||= 'unique_digest'.freeze
-  UNIQUE_ON_ALL_QUEUES_KEY ||= 'unique_on_all_queues'.freeze
-  UNIQUE_ARGS_ENABLED_KEY ||= 'unique_args_enabled'.freeze
+  ARGS                  = "args"
+  APARTMENT             = "apartment"
+  AT                    = "at"
+  CHANGELOGS            = "uniquejobs:changelog"
+  CLASS                 = "class"
+  CREATED_AT            = "created_at"
+  DEAD_VERSION          = "uniquejobs:dead"
+  DIGESTS               = "uniquejobs:digests"
+  EXPIRING_DIGESTS      = "uniquejobs:expiring_digests"
+  ERRORS                = "errors"
+  JID                   = "jid"
+  LIMIT                 = "limit"
+  LIVE_VERSION          = "uniquejobs:live"
+  LOCK                  = "lock"
+  LOCK_ARGS             = "lock_args"
+  LOCK_ARGS_METHOD      = "lock_args_method"
+  LOCK_DIGEST           = "lock_digest"
+  LOCK_EXPIRATION       = "lock_expiration"
+  LOCK_INFO             = "lock_info"
+  LOCK_LIMIT            = "lock_limit"
+  LOCK_PREFIX           = "lock_prefix"
+  LOCK_TIMEOUT          = "lock_timeout"
+  LOCK_TTL              = "lock_ttl"
+  LOCK_TYPE             = "lock_type"
+  ON_CLIENT_CONFLICT    = "on_client_conflict"
+  ON_CONFLICT           = "on_conflict"
+  ON_SERVER_CONFLICT    = "on_server_conflict"
+  PAYLOAD               = "payload"
+  PROCESSES             = "processes"
+  QUEUE                 = "queue"
+  RETRY                 = "retry"
+  SCHEDULE              = "schedule"
+  TIME                  = "time"
+  TIMEOUT               = "timeout"
+  TTL                   = "ttl"
+  TYPE                  = "type"
+  UNIQUE                = "unique"
+  UNIQUE_ACROSS_QUEUES  = "unique_across_queues"
+  UNIQUE_ACROSS_WORKERS = "unique_across_workers"
+  UNIQUE_ARGS           = "unique_args"
+  UNIQUE_ARGS_METHOD    = "unique_args_method"
+  UNIQUE_DIGEST         = "unique_digest"
+  UNIQUE_PREFIX         = "unique_prefix"
+  UNIQUE_REAPER         = "uniquejobs:reaper"
+  WORKER                = "worker"
 end

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -1,13 +1,85 @@
-begin
-  require 'active_support/core_ext/hash/keys'
-  require 'active_support/core_ext/hash/deep_merge'
-rescue LoadError
-  class Hash
+# frozen_string_literal: true
+
+# :nocov:
+
+#
+# Monkey patches for the ruby Hash
+#
+class Hash
+  unless {}.respond_to?(:slice)
+    #
+    # Returns only the matching keys in a new hash
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
-    end unless {}.respond_to?(:slice)
+    end
+  end
+
+  unless {}.respond_to?(:deep_stringify_keys)
+    #
+    # Depp converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_stringify_keys
+      deep_transform_keys(&:to_s)
+    end
+  end
+
+  unless {}.respond_to?(:deep_transform_keys)
+    #
+    # Deep transfor all keys by yielding to the caller
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def deep_transform_keys(&block)
+      _deep_transform_keys_in_object(self, &block)
+    end
+  end
+
+  unless {}.respond_to?(:stringify_keys)
+    #
+    # Converts all keys to string
+    #
+    #
+    # @return [Hash<String>]
+    #
+    def stringify_keys
+      transform_keys(&:to_s)
+    end
+  end
+
+  unless {}.respond_to?(:transform_keys)
+    #
+    # Transforms all keys by yielding to the caller
+    #
+    #
+    # @return [Hash]
+    #
+    def transform_keys
+      result = {}
+      each_key do |key|
+        result[yield(key)] = self[key]
+      end
+      result
+    end
+  end
 
+  unless {}.respond_to?(:slice!)
+    #
+    # Removes all keys not provided from the current hash and returns it
+    #
+    # @param [Array<String>, Array<Symbol>] keys the keys to match
+    #
+    # @return [Hash]
+    #
     def slice!(*keys)
       keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
       omit = slice(*self.keys - keys)
@@ -16,36 +88,45 @@ rescue LoadError
       hash.default_proc = default_proc if default_proc
       replace(hash)
       omit
-    end unless {}.respond_to?(:slice!)
+    end
   end
-end
 
-class Array
-  def extract_options!
-    if last.is_a?(Hash) && last.instance_of?(Hash)
-      pop
-    else
-      {}
+  private
+
+  unless {}.respond_to?(:_deep_transform_keys_in_object)
+    # support methods for deep transforming nested hashes and arrays
+    def _deep_transform_keys_in_object(object, &block)
+      case object
+      when Hash
+        object.each_with_object({}) do |(key, value), result|
+          result[yield(key)] = _deep_transform_keys_in_object(value, &block)
+        end
+      when Array
+        object.map { |element| _deep_transform_keys_in_object(element, &block) }
+      else
+        object
+      end
     end
-  end unless [].respond_to?(:extract_options!)
+  end
 end
 
-class String
-  def classify
-    camelize(sub(/.*\./, ''))
-  end unless ''.respond_to?(:classify)
-
-  def camelize(uppercase_first_letter = true)
-    string = self
-    string = if uppercase_first_letter
-               string.sub(/^[a-z\d]*/) { $&.capitalize }
-             else
-               string.sub(/^(?:(?=\b|[A-Z_])|\w)/) { $&.downcase }
-             end
-    string.gsub!(%r{(?:_|(\/))([a-z\d]*)}i) do
-      "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}"
-    end
-    string.gsub!(%r{/}, '::')
-    string
-  end unless ''.respond_to?(:camelize)
+#
+# Monkey patches for the ruby Array
+#
+class Array
+  unless [].respond_to?(:extract_options!)
+    #
+    # Extract the last argument if it is a hash
+    #
+    #
+    # @return [Hash]
+    #
+    def extract_options!
+      if last.is_a?(Hash) && last.instance_of?(Hash)
+        pop
+      else
+        {}
+      end
+    end
+  end
 end

data/lib/sidekiq_unique_jobs/deprecation.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Deprecation provides logging of deprecations
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Deprecation
+    #
+    # Mute warnings from this gem in a threaded context
+    #
+    #
+    # @return [void] <description>
+    #
+    # @yieldreturn [void]
+    def self.muted
+      orig_val = Thread.current[:uniquejobs_mute_deprecations]
+      Thread.current[:uniquejobs_mute_deprecations] = true
+      yield
+    ensure
+      Thread.current[:uniquejobs_mute_deprecations] = orig_val
+    end
+
+    #
+    # Check if deprecation warnings have been muted
+    #
+    #
+    # @return [true,false]
+    #
+    def self.muted?
+      Thread.current[:uniquejobs_mute_deprecations] == true
+    end
+
+    #
+    # Warn about deprecation
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      warn "DEPRECATION WARNING: #{msg}"
+      nil
+    end
+
+    #
+    # Warn about deprecation and provide a context
+    #
+    # @param [String] msg a descriptive reason for why the deprecation
+    #
+    # @return [void]
+    #
+    def self.warn_with_backtrace(msg)
+      return if SidekiqUniqueJobs::Deprecation.muted?
+
+      trace = "\n\nCALLED FROM:\n#{caller.join("\n")}"
+      warn(msg + trace)
+
+      nil
+    end
+  end
+end