sidekiq-unique-jobs 3.0.11 → 8.0.10

data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    # Locks jobs while executing
+    #   Locks from the server process
+    #   Unlocks after the server is done processing
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class WhileExecutingReject < WhileExecuting
+      # Overridden with a forced {OnConflict::Reject} strategy
+      # @return [OnConflict::Reject] a reject strategy
+      def server_strategy
+        @server_strategy ||= OnConflict.find_strategy(:reject).new(item, redis_pool)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock.rb

@@ -0,0 +1,348 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Lock provides access to information about a lock
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Lock # rubocop:disable Metrics/ClassLength
+    # includes "SidekiqUniqueJobs::Connection"
+    # @!parse include SidekiqUniqueJobs::Connection
+    include SidekiqUniqueJobs::Connection
+
+    # includes "SidekiqUniqueJobs::Timing"
+    # @!parse include SidekiqUniqueJobs::Timing
+    include SidekiqUniqueJobs::Timing
+
+    # includes "SidekiqUniqueJobs::JSON"
+    # @!parse include SidekiqUniqueJobs::JSON
+    include SidekiqUniqueJobs::JSON
+
+    #
+    # @!attribute [r] key
+    #   @return [String] the entity redis key
+    attr_reader :key
+
+    #
+    # Initialize a locked lock
+    #
+    # @param [String] digest a unique digest
+    # @param [String] job_id a sidekiq JID
+    # @param [Hash] lock_info information about the lock
+    #
+    # @return [Lock] a newly lock that has been locked
+    #
+    def self.create(digest, job_id, lock_info: {}, time: Timing.now_f, score: nil)
+      lock = new(digest, time: time)
+      lock.lock(job_id, lock_info, score)
+      lock
+    end
+
+    #
+    # Initialize a new lock
+    #
+    # @param [String, Key] key either a digest or an instance of a {Key}
+    # @param [Timstamp, Float] time nil optional timestamp to initiate this lock with
+    #
+    def initialize(key, time: nil)
+      @key = get_key(key)
+      time = time.to_f unless time.is_a?(Float)
+      return unless time.nonzero?
+
+      @created_at = time
+    end
+
+    #
+    # Locks a job_id
+    #
+    # @note intended only for testing purposes
+    #
+    # @param [String] job_id a sidekiq JID
+    # @param [Hash] lock_info information about the lock
+    #
+    # @return [void]
+    #
+    def lock(job_id, lock_info = {}, score = nil)
+      score ||= now_f
+      redis do |conn|
+        conn.multi do |pipeline|
+          pipeline.set(key.digest, job_id)
+          pipeline.hset(key.locked, job_id, now_f)
+          info.set(lock_info, pipeline)
+          add_digest_to_set(pipeline, lock_info, score)
+          pipeline.zadd(key.changelog, score, changelog_json(job_id, "queue.lua", "Queued"))
+          pipeline.zadd(key.changelog, score, changelog_json(job_id, "lock.lua", "Locked"))
+        end
+      end
+    end
+
+    #
+    # Create the :QUEUED key
+    #
+    # @note intended only for testing purposes
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [void]
+    #
+    def queue(job_id)
+      redis do |conn|
+        conn.lpush(key.queued, job_id)
+      end
+    end
+
+    #
+    # Create the :PRIMED key
+    #
+    # @note intended only for testing purposes
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [void]
+    #
+    def prime(job_id)
+      redis do |conn|
+        conn.lpush(key.primed, job_id)
+      end
+    end
+
+    #
+    # Unlock a specific job_id
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [true] when job_id was removed
+    # @return [false] when job_id wasn't locked
+    #
+    def unlock(job_id)
+      locked.del(job_id)
+    end
+
+    #
+    # Deletes all the redis keys for this lock
+    #
+    #
+    # @return [Integer] the number of keys deleted in redis
+    #
+    def del
+      redis do |conn|
+        conn.multi do |pipeline|
+          pipeline.zrem(DIGESTS, key.digest)
+          pipeline.del(key.digest, key.queued, key.primed, key.locked, key.info)
+        end
+      end
+    end
+
+    #
+    # Returns either the time the lock was initialized with or
+    #   the first changelog entry's timestamp
+    #
+    #
+    # @return [Float] a floaty timestamp represantation
+    #
+    def created_at
+      @created_at ||= changelogs.first&.[]("time")
+    end
+
+    #
+    # Returns all job_id's for this lock
+    #
+    # @note a JID can be present in 3 different places
+    #
+    #
+    # @return [Array<String>] an array with JIDs
+    #
+    def all_jids
+      (queued_jids + primed_jids + locked_jids).uniq
+    end
+
+    #
+    # Returns a collection of locked job_id's
+    #
+    # @param [true, false] with_values false provide the timestamp for the lock
+    #
+    # @return [Hash<String, Float>] when given `with_values: true`
+    # @return [Array<String>] when given `with_values: false`
+    #
+    def locked_jids(with_values: false)
+      locked.entries(with_values: with_values)
+    end
+
+    #
+    # Returns the queued JIDs
+    #
+    #
+    # @return [Array<String>] an array with queued job_ids
+    #
+    def queued_jids
+      queued.entries
+    end
+
+    #
+    # Returns the primed JIDs
+    #
+    #
+    # @return [Array<String>] an array with primed job_ids
+    #
+    def primed_jids
+      primed.entries
+    end
+
+    #
+    # Returns all matching changelog entries for this lock
+    #
+    #
+    # @return [Array<Hash>] an array with changelogs
+    #
+    def changelogs
+      changelog.entries(pattern: "*#{key.digest}*")
+    end
+
+    #
+    # The digest key
+    #
+    # @note Used for exists checks to avoid enqueuing
+    #   the same lock twice
+    #
+    #
+    # @return [Redis::String] a string representation of the key
+    #
+    def digest
+      @digest ||= Redis::String.new(key.digest)
+    end
+
+    #
+    # The queued list
+    #
+    #
+    # @return [Redis::List] for queued JIDs
+    #
+    def queued
+      @queued ||= Redis::List.new(key.queued)
+    end
+
+    #
+    # The primed list
+    #
+    #
+    # @return [Redis::List] for primed JIDs
+    #
+    def primed
+      @primed ||= Redis::List.new(key.primed)
+    end
+
+    #
+    # The locked hash
+    #
+    #
+    # @return [Redis::Hash] for locked JIDs
+    #
+    def locked
+      @locked ||= Redis::Hash.new(key.locked)
+    end
+
+    #
+    # Information about the lock
+    #
+    #
+    # @return [Redis::Hash] with lock information
+    #
+    def info
+      @info ||= LockInfo.new(key.info)
+    end
+
+    #
+    # A sorted set with changelog entries
+    #
+    # @see Changelog for more information
+    #
+    #
+    # @return [Changelog]
+    #
+    def changelog
+      @changelog ||= Changelog.new
+    end
+
+    #
+    # A nicely formatted string with information about this lock
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      <<~MESSAGE
+        Lock status for #{key}
+
+                  value: #{digest.value}
+                   info: #{info.value}
+            queued_jids: #{queued_jids}
+            primed_jids: #{primed_jids}
+            locked_jids: #{locked_jids}
+             changelogs: #{changelogs}
+      MESSAGE
+    end
+
+    #
+    # @see to_s
+    #
+    def inspect
+      to_s
+    end
+
+    private
+
+    #
+    # Ensure the key is a {Key}
+    #
+    # @param [String, Key] key
+    #
+    # @return [Key]
+    #
+    def get_key(key)
+      if key.is_a?(SidekiqUniqueJobs::Key)
+        key
+      else
+        SidekiqUniqueJobs::Key.new(key)
+      end
+    end
+
+    #
+    # Generate a changelog entry for the given arguments
+    #
+    # @param [String] job_id a sidekiq JID
+    # @param [String] script the name of the script generating this entry
+    # @param [String] message a descriptive message for later review
+    #
+    # @return [String] a JSON string matching the Lua script structure
+    #
+    def changelog_json(job_id, script, message)
+      dump_json(
+        digest: key.digest,
+        job_id: job_id,
+        script: script,
+        message: message,
+        time: now_f,
+      )
+    end
+
+    #
+    # Add the digest to the correct sorted set
+    #
+    # @param [Object] pipeline a redis pipeline object for issue commands
+    # @param [Hash] lock_info the lock info relevant to the digest
+    #
+    # @return [nil]
+    #
+    def add_digest_to_set(pipeline, lock_info, score = nil)
+      score ||= now_f
+      digest_string = key.digest
+
+      if lock_info["lock"] == :until_expired
+        pipeline.zadd(key.expiring_digests, score + lock_info["ttl"], digest_string)
+      else
+        pipeline.zadd(key.digests, score, digest_string)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock_args.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles uniqueness of sidekiq arguments
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class LockArgs
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::SidekiqWorkerMethods
+    include SidekiqUniqueJobs::JSON
+
+    # Convenience method for returning a digest
+    # @param [Hash] item a Sidekiq job hash
+    # @return [String] a unique digest
+    def self.call(item)
+      new(item).lock_args
+    end
+
+    # The sidekiq job hash
+    # @return [Hash] the Sidekiq job hash
+    attr_reader :item
+    #
+    # @!attribute [r] args
+    #   @return [Array<Objet>] the arguments passed to `perform_async`
+    attr_reader :args
+
+    # @param [Hash] item a Sidekiq job hash
+    def initialize(item)
+      @item = item
+      @args = item[ARGS]
+      self.job_class = item[CLASS]
+    end
+
+    # The unique arguments to use for creating a lock
+    # @return [Array] the arguments filters by the {#filtered_args} method if {#lock_args_enabled?}
+    def lock_args
+      @lock_args ||= filtered_args || []
+    end
+
+    # Checks if the worker class has enabled lock_args
+    # @return [true, false]
+    def lock_args_enabled?
+      # return false unless lock_args_method_valid?
+
+      lock_args_method
+    end
+
+    # Validate that the lock_args_method is acceptable
+    # @return [true, false]
+    def lock_args_method_valid?
+      [NilClass, TrueClass, FalseClass].none? { |klass| lock_args_method.is_a?(klass) }
+    end
+
+    # Checks if the worker class has disabled lock_args
+    # @return [true, false]
+    def lock_args_disabled?
+      !lock_args_method
+    end
+
+    # Filters unique arguments by proc or symbol
+    # @return [Array] {#filter_by_proc} when {#lock_args_method} is a Proc
+    # @return [Array] {#filter_by_symbol} when {#lock_args_method} is a Symbol
+    # @return [Array] args unfiltered when neither of the above
+    def filtered_args
+      return args if lock_args_disabled?
+
+      json_args = Normalizer.jsonify(args)
+
+      case lock_args_method
+      when Proc
+        filter_by_proc(json_args)
+      when Symbol
+        filter_by_symbol(json_args)
+      end
+    end
+
+    # Filters unique arguments by proc configured in the sidekiq worker
+    # @param [Array] args the arguments passed to the sidekiq worker
+    # @return [Array] with the filtered arguments
+    def filter_by_proc(args)
+      lock_args_method.call(args)
+    end
+
+    # Filters unique arguments by method configured in the sidekiq worker
+    # @param [Array] args the arguments passed to the sidekiq worker
+    # @return [Array] unfiltered unless {#job_method_defined?}
+    # @return [Array] with the filtered arguments
+    def filter_by_symbol(args)
+      return args unless job_method_defined?(lock_args_method)
+
+      job_class.send(lock_args_method, args)
+    rescue ArgumentError
+      raise SidekiqUniqueJobs::InvalidUniqueArguments,
+            given: args,
+            job_class: job_class,
+            lock_args_method: lock_args_method
+    end
+
+    # The method to use for filtering unique arguments
+    def lock_args_method
+      @lock_args_method ||= job_options.slice(LOCK_ARGS_METHOD, UNIQUE_ARGS_METHOD).values.first
+      @lock_args_method ||= :lock_args if job_method_defined?(:lock_args)
+      @lock_args_method ||= :unique_args if job_method_defined?(:unique_args)
+      @lock_args_method ||= default_lock_args_method
+    end
+
+    # The global worker options defined in Sidekiq directly
+    def default_lock_args_method
+      default_job_options[LOCK_ARGS_METHOD] ||
+        default_job_options[UNIQUE_ARGS_METHOD]
+    end
+
+    #
+    # The globally default worker options configured from Sidekiq
+    #
+    #
+    # @return [Hash<String, Object>]
+    #
+    def default_job_options
+      @default_job_options ||= if Sidekiq.respond_to?(:default_job_options)
+        Sidekiq.default_job_options.stringify_keys
+      else
+        Sidekiq.default_worker_options.stringify_keys
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Gathers all configuration for a lock
+  #   which helps reduce the amount of instance variables
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class LockConfig
+    #
+    # @!attribute [r] type
+    #   @return [Symbol] the type of lock
+    attr_reader :type
+    #
+    # @!attribute [r] job
+    #   @return [Symbol] the job class
+    attr_reader :job
+    #
+    # @!attribute [r] limit
+    #   @return [Integer] the number of simultaneous locks
+    attr_reader :limit
+    #
+    # @!attribute [r] timeout
+    #   @return [Integer, nil] the time to wait for a lock
+    attr_reader :timeout
+    #
+    # @!attribute [r] ttl
+    #   @return [Integer, nil] the time (in seconds) to live after successful
+    attr_reader :ttl
+    #
+    # @!attribute [r] ttl
+    #   @return [Integer, nil] the time (in milliseconds) to live after successful
+    attr_reader :pttl
+    #
+    # @!attribute [r] lock_info
+    #   @return [Boolean] indicate wether to use lock_info or not
+    attr_reader :lock_info
+    #
+    # @!attribute [r] on_conflict
+    #   @return [Symbol, Hash<Symbol, Symbol>] the strategies to use as conflict resolution
+    attr_reader :on_conflict
+    #
+    # @!attribute [r] errors
+    #   @return [Array<Hash<Symbol, Array<String>] a collection of configuration errors
+    attr_reader :errors
+
+    #
+    # Instantiate a new lock_config based on sidekiq options in worker
+    #
+    # @param [Hash] options sidekiq_options for worker
+    #
+    # @return [LockConfig]
+    #
+    def self.from_worker(options)
+      new(options.deep_stringify_keys)
+    end
+
+    def initialize(job_hash = {})
+      @type        = job_hash[LOCK]&.to_sym
+      @job         = SidekiqUniqueJobs.safe_constantize(job_hash[CLASS])
+      @limit       = job_hash.fetch(LOCK_LIMIT, 1)&.to_i
+      @timeout     = job_hash.fetch(LOCK_TIMEOUT, 0)&.to_i
+      @ttl         = job_hash.fetch(LOCK_TTL) { job_hash.fetch(LOCK_EXPIRATION, nil) }.to_i
+      @pttl        = ttl * 1_000
+      @lock_info   = job_hash.fetch(LOCK_INFO) { SidekiqUniqueJobs.config.lock_info }
+      @on_conflict = job_hash.fetch(ON_CONFLICT, nil)
+      @errors      = job_hash.fetch(ERRORS) { {} }
+
+      @on_client_conflict = job_hash[ON_CLIENT_CONFLICT]
+      @on_server_conflict = job_hash[ON_SERVER_CONFLICT]
+    end
+
+    def lock_info?
+      lock_info
+    end
+
+    #
+    # Indicate if timeout was set
+    #
+    #
+    # @return [true,false]
+    #
+    def wait_for_lock?
+      timeout.nil? || timeout.positive?
+    end
+
+    #
+    # Is the configuration valid?
+    #
+    #
+    # @return [true,false]
+    #
+    def valid?
+      errors.empty?
+    end
+
+    #
+    # Return a nice descriptive message with all validation errors
+    #
+    #
+    # @return [String]
+    #
+    def errors_as_string
+      return if valid?
+
+      @errors_as_string ||= begin
+        error_msg = +"\t"
+        error_msg << errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")
+        error_msg
+      end
+    end
+
+    # the strategy to use as conflict resolution from sidekiq client
+    def on_client_conflict
+      @on_client_conflict ||= if on_conflict.is_a?(Hash)
+        on_conflict["client"] || on_conflict[:client]
+      else
+        on_conflict
+      end
+    end
+
+    # the strategy to use as conflict resolution from sidekiq server
+    def on_server_conflict
+      @on_server_conflict ||= if on_conflict.is_a?(Hash)
+        on_conflict["server"] || on_conflict[:server]
+      else
+        on_conflict
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock_digest.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+module SidekiqUniqueJobs
+  # Handles uniqueness of sidekiq arguments
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class LockDigest
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::JSON
+    include SidekiqUniqueJobs::SidekiqWorkerMethods
+
+    #
+    # Generates a new digest
+    #
+    # @param [Hash] item a sidekiq job hash
+    #
+    # @return [String] a unique digest for the given arguments
+    #
+    def self.call(item)
+      new(item).lock_digest
+    end
+
+    # The sidekiq job hash
+    # @return [Hash] the Sidekiq job hash
+    attr_reader :item
+    #
+    # @!attribute [r] args
+    #   @return [Array<Objet>] the arguments passed to `perform_async`
+    attr_reader :lock_args
+    #
+    # @!attribute [r] args
+    #   @return [String] the prefix for the unique key
+    attr_reader :lock_prefix
+
+    # @param [Hash] item a Sidekiq job hash
+    def initialize(item)
+      @item        = item
+      @lock_args   = item[LOCK_ARGS] || item[UNIQUE_ARGS] # TODO: Deprecate UNIQUE_ARGS
+      @lock_prefix = item[LOCK_PREFIX] || item[UNIQUE_PREFIX] # TODO: Deprecate UNIQUE_PREFIX
+      self.job_class = item[CLASS]
+    end
+
+    # Memoized lock_digest
+    # @return [String] a unique digest
+    def lock_digest
+      @lock_digest ||= create_digest
+    end
+
+    # Creates a namespaced unique digest based on the {#digestable_hash} and the {#lock_prefix}
+    # @return [String] a unique digest
+    def create_digest
+      digest = OpenSSL::Digest::MD5.hexdigest(dump_json(digestable_hash.sort))
+      "#{lock_prefix}:#{digest}"
+    end
+
+    # Filter a hash to use for digest
+    # @return [Hash] to use for digest
+    def digestable_hash
+      @item.slice(CLASS, QUEUE, LOCK_ARGS, APARTMENT).tap do |hash|
+        hash.delete(QUEUE) if unique_across_queues?
+        hash.delete(CLASS) if unique_across_workers?
+      end
+    end
+
+    # Checks if we should disregard the queue when creating the unique digest
+    # @return [true, false]
+    def unique_across_queues?
+      item[UNIQUE_ACROSS_QUEUES] || job_options[UNIQUE_ACROSS_QUEUES]
+    end
+
+    # Checks if we should disregard the worker when creating the unique digest
+    # @return [true, false]
+    def unique_across_workers?
+      item[UNIQUE_ACROSS_WORKERS] || job_options[UNIQUE_ACROSS_WORKERS]
+    end
+  end
+end