sidekiq-unique-jobs 4.0.18 → 7.1.28

data/lib/sidekiq_unique_jobs/digests.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Digests < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
+    DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
+    SCAN_PATTERN  = "*"
+
+    def initialize(digests_key = DIGESTS)
+      super(digests_key)
+    end
+
+    #
+    # Adds a digest
+    #
+    # @param [String] digest the digest to add
+    #
+    def add(digest)
+      redis { |conn| conn.zadd(key, now_f, digest) }
+    end
+
+    # Deletes unique digests by pattern
+    #
+    # @param [String] pattern a key pattern to match with
+    # @param [Integer] count the maximum number
+    # @return [Hash<String,Float>] Hash mapping of digest matching the given pattern and score
+
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = entries(pattern: pattern, count: count).keys
+        redis { |conn| BatchDelete.call(digests, conn) }
+      end
+
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    # Delete unique digests by digest
+    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
+    #
+    # @param [String] digest a unique digest to delete
+    def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
+      result, elapsed = timed do
+        call_script(:delete_by_digest, [
+                      digest,
+                      "#{digest}:QUEUED",
+                      "#{digest}:PRIMED",
+                      "#{digest}:LOCKED",
+                      "#{digest}:RUN",
+                      "#{digest}:RUN:QUEUED",
+                      "#{digest}:RUN:PRIMED",
+                      "#{digest}:RUN:LOCKED",
+                      key,
+                    ])
+      end
+
+      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    #
+    # The entries in this sorted set
+    #
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] count DEFAULT_COUNT the number of entries to return
+    #
+    # @return [Array<String>] an array of digests matching the given pattern
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      options = {}
+      options[:match] = pattern
+      options[:count] = count
+
+      redis { |conn| conn.zscan_each(key, **options).to_a }.to_h
+    end
+
+    #
+    # Returns a paginated
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern SCAN_PATTERN the match pattern to search for
+    # @param [Integer] page_size 100 the size per page
+    #
+    # @return [Array<Integer, Integer, Array<Lock>>] total_size, next_cursor, locks
+    #
+    def page(cursor: 0, pattern: SCAN_PATTERN, page_size: 100)
+      redis do |conn|
+        total_size, digests = conn.multi do |pipeline|
+          pipeline.zcard(key)
+          pipeline.zscan(key, cursor, match: pattern, count: page_size)
+        end
+
+        [
+          total_size.to_i,
+          digests[0].to_i, # next_cursor
+          digests[1].map { |digest, score| Lock.new(digest, time: score) }, # entries
+        ]
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Base class for all exceptions raised from the gem
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UniqueJobsError < ::RuntimeError
+  end
+
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Conflict < UniqueJobsError
+    def initialize(item)
+      super("Item with the key: #{item[LOCK_DIGEST]} is already scheduled or processing")
+    end
+  end
+
+  #
+  # Raised when no block was given
+  #
+  class NoBlockGiven < SidekiqUniqueJobs::UniqueJobsError; end
+
+  #
+  # Raised when a notification has been mistyped
+  #
+  class NoSuchNotificationError < UniqueJobsError; end
+
+  #
+  # Error raised when trying to add a duplicate lock
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateLock < UniqueJobsError
+  end
+
+  #
+  # Error raised when trying to add a duplicate stragegy
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class DuplicateStrategy < UniqueJobsError
+  end
+
+  #
+  # Error raised when an invalid argument is given
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidArgument < UniqueJobsError
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class InvalidWorker < UniqueJobsError
+    def initialize(lock_config)
+      super(<<~FAILURE_MESSAGE)
+        Expected #{lock_config.worker} to have valid sidekiq options but found the following problems:
+        #{lock_config.errors_as_string}
+      FAILURE_MESSAGE
+    end
+  end
+
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class InvalidUniqueArguments < UniqueJobsError
+    def initialize(options)
+      given            = options[:given]
+      job_class        = options[:job_class]
+      lock_args_method = options[:lock_args_method]
+      lock_args_meth   = job_class.method(lock_args_method)
+      num_args         = lock_args_meth.arity
+      source_location  = lock_args_meth.source_location
+
+      super(
+        "#{job_class}##{lock_args_method} takes #{num_args} arguments, received #{given.inspect}" \
+        "\n\n" \
+        "   #{source_location.join(':')}"
+      )
+    end
+  end
+
+  #
+  # Raised when a workers configuration is invalid
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class NotUniqueWorker < UniqueJobsError
+    def initialize(options)
+      super("#{options[:class]} is not configured for uniqueness. Missing the key `:lock` in #{options.inspect}")
+    end
+  end
+
+  # Error raised from {OptionsWithFallback#lock_class}
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class UnknownLock < UniqueJobsError
+  end
+end

data/lib/sidekiq_unique_jobs/expiring_digests.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class ExpiringDigests provides access to the expiring digests used by until_expired locks
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class ExpiringDigests < Digests
+    def initialize
+      super(EXPIRING_DIGESTS)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/job.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Utility class to append uniqueness to the sidekiq job hash
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Job
+    extend self
+
+    # Adds timeout, expiration, lock_args, lock_prefix and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def prepare(item)
+      stringify_on_conflict_hash(item)
+      add_lock_timeout(item)
+      add_lock_ttl(item)
+      add_digest(item)
+    end
+
+    # Adds lock_args, lock_prefix and lock_digest to the sidekiq job hash
+    # @return [Hash] the job hash
+    def add_digest(item)
+      add_lock_prefix(item)
+      add_lock_args(item)
+      add_lock_digest(item)
+
+      item
+    end
+
+    private
+
+    def stringify_on_conflict_hash(item)
+      on_conflict = item[ON_CONFLICT]
+      return unless on_conflict.is_a?(Hash)
+
+      item[ON_CONFLICT] = on_conflict.deep_stringify_keys
+    end
+
+    def add_lock_ttl(item)
+      item[LOCK_TTL] = SidekiqUniqueJobs::LockTTL.calculate(item)
+    end
+
+    def add_lock_timeout(item)
+      item[LOCK_TIMEOUT] ||= SidekiqUniqueJobs::LockTimeout.calculate(item)
+    end
+
+    def add_lock_args(item)
+      item[LOCK_ARGS] ||= SidekiqUniqueJobs::LockArgs.call(item)
+    end
+
+    def add_lock_digest(item)
+      item[LOCK_DIGEST] ||= SidekiqUniqueJobs::LockDigest.call(item)
+    end
+
+    def add_lock_prefix(item)
+      item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.lock_prefix
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/json.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles loading and dumping of json
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module JSON
+    module_function
+
+    #
+    # Parses a JSON string into an object
+    #
+    # @param [String] string the object to parse
+    #
+    # @return [Object]
+    #
+    def load_json(string)
+      return if string.nil? || string.empty?
+
+      ::JSON.parse(string)
+    end
+
+    #
+    # Prevents trying JSON.load from raising errors given argument is a hash
+    #
+    # @param [String, Hash] string the JSON string to parse
+    #
+    # @return [Hash,Array]
+    #
+    def safe_load_json(string)
+      return string if string.is_a?(Hash)
+
+      load_json(string)
+    end
+
+    #
+    # Dumps an object into a JSON string
+    #
+    # @param [Object] object a JSON convertible object
+    #
+    # @return [String] a JSON string
+    #
+    def dump_json(object)
+      ::JSON.generate(object)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/key.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Key class wraps logic dealing with various lock keys
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Key
+    #
+    # @!attribute [r] digest
+    #   @return [String] the digest key for which keys are created
+    attr_reader :digest
+    #
+    # @!attribute [r] queued
+    #   @return [String] the list key with queued job_id's
+    attr_reader :queued
+    #
+    # @!attribute [r] primed
+    #   @return [String] the list key with primed job_id's
+    attr_reader :primed
+    #
+    # @!attribute [r] locked
+    #   @return [String] the hash key with locked job_id's
+    attr_reader :locked
+    #
+    # @!attribute [r] info
+    #   @return [String] information about the lock
+    attr_reader :info
+    #
+    # @!attribute [r] changelog
+    #   @return [String] the zset with changelog entries
+    attr_reader :changelog
+    #
+    # @!attribute [r] digests
+    #   @return [String] the zset with locked digests
+    attr_reader :digests
+    #
+    # @!attribute [r] expiring_digests
+    #   @return [String] the zset with locked expiring_digests
+    attr_reader :expiring_digests
+
+    #
+    # Initialize a new Key
+    #
+    # @param [String] digest the digest to use as key
+    #
+    def initialize(digest)
+      @digest           = digest
+      @queued           = suffixed_key("QUEUED")
+      @primed           = suffixed_key("PRIMED")
+      @locked           = suffixed_key("LOCKED")
+      @info             = suffixed_key("INFO")
+      @changelog        = CHANGELOGS
+      @digests          = DIGESTS
+      @expiring_digests = EXPIRING_DIGESTS
+    end
+
+    #
+    # Provides the only important information about this keys
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      digest
+    end
+
+    # @see to_s
+    def inspect
+      digest
+    end
+
+    #
+    # Compares keys by digest
+    #
+    # @param [Key] other the key to compare with
+    #
+    # @return [true, false]
+    #
+    def ==(other)
+      digest == other.digest
+    end
+
+    #
+    # Returns all keys as an ordered array
+    #
+    # @return [Array] an ordered array with all keys
+    #
+    def to_a
+      [digest, queued, primed, locked, info, changelog, digests, expiring_digests]
+    end
+
+    private
+
+    def suffixed_key(variable)
+      "#{digest}:#{variable}"
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Abstract base class for locks
+    #
+    # @abstract
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class BaseLock
+      extend Forwardable
+
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::Logging
+
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
+
+      #
+      # Validates that the sidekiq_options for the worker is valid
+      #
+      # @param [Hash] options the sidekiq_options given to the worker
+      #
+      # @return [void]
+      #
+      def self.validate_options(options = {})
+        Validator.validate(options)
+      end
+
+      # NOTE: Mainly used for a clean testing API
+      #
+      def_delegators :locksmith, :locked?
+
+      # @param [Hash] item the Sidekiq job hash
+      # @param [Proc] callback the callback to use after unlock
+      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+      def initialize(item, callback, redis_pool = nil)
+        @item       = item
+        @callback   = callback
+        @redis_pool = redis_pool
+        @attempt    = 0
+        prepare_item # Used to ease testing
+        @lock_config = LockConfig.new(item)
+      end
+
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
+      end
+
+      # Execute the job in the Sidekiq server processor
+      # @raise [NotImplementedError] needs to be implemented in child class
+      def execute
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
+      end
+
+      #
+      # The lock manager/client
+      #
+      # @api private
+      # @return [SidekiqUniqueJobs::Locksmith] the locksmith for this sidekiq job
+      #
+      def locksmith
+        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
+      end
+
+      private
+
+      # @!attribute [r] item
+      #   @return [Hash<String, Object>] the Sidekiq job hash
+      attr_reader :item
+      # @!attribute [r] lock_config
+      #   @return [LockConfig] a lock configuration
+      attr_reader :lock_config
+      # @!attribute [r] redis_pool
+      #   @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      attr_reader :redis_pool
+      # @!attribute [r] callback
+      #   @return [Proc] the block to call after unlock
+      attr_reader :callback
+      # @!attribute [r] attempt
+      #   @return [Integer] the current locking attempt
+      attr_reader :attempt
+
+      #
+      # Eases testing by allowing the lock implementation to add the missing
+      # keys to the job hash.
+      #
+      #
+      # @return [void] the return value should be irrelevant
+      #
+      def prepare_item
+        return if item.key?(LOCK_DIGEST)
+
+        # The below should only be done to ease testing
+        # in production this will be done by the middleware
+        SidekiqUniqueJobs::Job.prepare(item)
+      end
+
+      #
+      # Call whatever strategry that has been configured
+      #
+      # @param [Symbol] origin: the origin `:client` or `:server`
+      #
+      # @return [void] the return value is irrelevant
+      #
+      # @yieldparam [void] if a new job id was set and a block is given
+      # @yieldreturn [void] the yield is irrelevant, it only provides a mechanism in
+      #   one specific situation to yield back to the middleware.
+      def call_strategy(origin:)
+        new_job_id = nil
+        strategy   = strategy_for(origin)
+        @attempt  += 1
+
+        strategy.call { new_job_id = lock if strategy.replace? && @attempt < 2 }
+        yield if new_job_id && block_given?
+      end
+
+      def unlock_and_callback
+        return callback_safely if locksmith.unlock
+
+        reflect(:unlock_failed, item)
+      end
+
+      def callback_safely
+        callback&.call
+        item[JID]
+      rescue StandardError
+        reflect(:after_unlock_callback_failed, item)
+        raise
+      end
+
+      def strategy_for(origin)
+        case origin
+        when :client
+          client_strategy
+        when :server
+          server_strategy
+        else
+          raise SidekiqUniqueJobs::InvalidArgument,
+                "#origin needs to be either `:server` or `:client`"
+        end
+      end
+
+      def client_strategy
+        @client_strategy ||=
+          OnConflict.find_strategy(lock_config.on_client_conflict).new(item, redis_pool)
+      end
+
+      def server_strategy
+        @server_strategy ||=
+          OnConflict.find_strategy(lock_config.on_server_conflict).new(item, redis_pool)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/client_validator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq client process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ClientValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:raise, :reject, :reschedule].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq client process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_client_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_client_conflict] = "#{on_conflict} is incompatible with the client process"
+        lock_config
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/server_validator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    #
+    # Validates the sidekiq options for the Sidekiq server process
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class ServerValidator
+      #
+      # @return [Array<Symbol>] a collection of invalid conflict resolutions
+      INVALID_ON_CONFLICTS = [:replace].freeze
+
+      #
+      # Validates the sidekiq options for the Sidekiq server process
+      #
+      #
+      def self.validate(lock_config)
+        on_conflict = lock_config.on_server_conflict
+        return lock_config unless INVALID_ON_CONFLICTS.include?(on_conflict)
+
+        lock_config.errors[:on_server_conflict] = "#{on_conflict} is incompatible with the server process"
+      end
+    end
+  end
+end